2024-04-02T04:35:35.564 INFO:root:teuthology version: 0.0.1.dev262+ge691533 2024-04-02T04:35:35.564 DEBUG:teuthology.run:Teuthology command: teuthology --owner scheduled_teuthology@teuthology --description orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/repo_digest 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} --name teuthology-2024-04-01_21:08:08-orch-squid-distro-default-smithi --verbose --archive /home/teuthworker/archive/teuthology-2024-04-01_21:08:08-orch-squid-distro-default-smithi/7634859 -- /home/teuthworker/archive/teuthology-2024-04-01_21:08:08-orch-squid-distro-default-smithi/7634859/orig.config.yaml 2024-04-02T04:35:35.610 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-02T04:35:35.700 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-01_21:08:08-orch-squid-distro-default-smithi/7634859 branch: squid description: orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/repo_digest 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} email: ceph-qa@ceph.com first_in_suite: false job_id: '7634859' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi name: teuthology-2024-04-01_21:08:08-orch-squid-distro-default-smithi no_nested_subset: false nuke-on-error: true os_type: centos os_version: 9.stream overrides: admin_socket: branch: squid ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mgr/cephadm/use_agent: false mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 6cec793d9073f0594e2364b9b687ecaf22424cca ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 6cec793d9073f0594e2364b9b687ecaf22424cca workunit: branch: squid sha1: 6cec793d9073f0594e2364b9b687ecaf22424cca owner: scheduled_teuthology@teuthology priority: 100 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b seed: 6530 sha1: 6cec793d9073f0594e2364b9b687ecaf22424cca sleep_before_teardown: 0 subset: 52/64 suite: orch suite_branch: squid suite_path: /home/teuthworker/src/git.ceph.com_ceph_6cec793d9073f0594e2364b9b687ecaf22424cca/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 6cec793d9073f0594e2364b9b687ecaf22424cca targets: smithi086.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBD6U1xsGHi5TYjDNNLsRfTlkPPq3bmTd/DfNTpWlI/gq/At+1k54Bp4jzuZ+GEic4wDmCv31U+JoPc4HUo7Llgo= smithi192.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHRRJNkbXvbeEt5uCG7WHBMoLFh1on7lkBXYMBfbltGylPGKF3sF4X+1aN2bwtLdAcXFQ1RRPJ2ckWMakwO0nwY= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 - cephadm.shell: mon.a: - ceph config set mgr mgr/cephadm/use_repo_digest false --force - cephadm.shell: env: - sha1 mon.a: - radosgw-admin realm create --rgw-realm=r --default - radosgw-admin zonegroup create --rgw-zonegroup=default --master --default - radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default - radosgw-admin period update --rgw-realm=r --commit - ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000 - ceph osd pool create foo - rbd pool init foo - ceph orch apply iscsi foo u p - sleep 180 - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph orch ps - ceph versions - ceph -s - ceph orch ls - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 2' - ceph mgr fail - sleep 180 - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph health detail - ceph -s - ceph mgr fail - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 1' - ceph mgr fail - sleep 180 - ceph orch ps - ceph orch ls - ceph versions - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph versions | jq -e '.mgr | length == 1' - ceph versions | jq -e '.mgr | keys' | grep $sha1 - ceph versions | jq -e '.overall | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 1' - ceph versions | jq -e '.mon | keys' | grep $sha1 - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 5' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 7' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 8' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 1' - ceph versions | jq -e '.osd | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.rgw | length == 1' - ceph versions | jq -e '.rgw | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 mon.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph versions - ceph orch upgrade status - ceph health detail - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - ceph orch ls | grep '^osd ' - cephadm.shell: mon.a: - ceph orch upgrade ls - ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0 - ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2 teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: e691533f9cbb33d85b2187bba20d7102f098636d timestamp: 2024-04-01_21:08:08 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-02T04:35:35.700 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_6cec793d9073f0594e2364b9b687ecaf22424cca/qa; will attempt to use it 2024-04-02T04:35:35.700 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_6cec793d9073f0594e2364b9b687ecaf22424cca/qa/tasks 2024-04-02T04:35:35.701 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-02T04:35:35.702 INFO:teuthology.task.internal:Checking packages... 2024-04-02T04:35:35.723 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '6cec793d9073f0594e2364b9b687ecaf22424cca' 2024-04-02T04:35:35.723 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-02T04:35:35.724 INFO:teuthology.packaging:ref: None 2024-04-02T04:35:35.724 INFO:teuthology.packaging:tag: None 2024-04-02T04:35:35.724 INFO:teuthology.packaging:branch: squid 2024-04-02T04:35:35.724 INFO:teuthology.packaging:sha1: 6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T04:35:35.724 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-04-02T04:35:35.916 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-1979.g6cec793d 2024-04-02T04:35:35.917 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-02T04:35:35.924 INFO:teuthology.task.internal:no buildpackages task found 2024-04-02T04:35:35.924 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-02T04:35:35.931 INFO:teuthology.task.internal:Saving configuration 2024-04-02T04:35:35.948 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-02T04:35:35.954 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-02T04:35:35.979 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi086.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-01_21:08:08-orch-squid-distro-default-smithi/7634859', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-02 04:30:39.916834', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBD6U1xsGHi5TYjDNNLsRfTlkPPq3bmTd/DfNTpWlI/gq/At+1k54Bp4jzuZ+GEic4wDmCv31U+JoPc4HUo7Llgo='} 2024-04-02T04:35:36.005 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi192.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-01_21:08:08-orch-squid-distro-default-smithi/7634859', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-02 04:30:39.915163', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHRRJNkbXvbeEt5uCG7WHBMoLFh1on7lkBXYMBfbltGylPGKF3sF4X+1aN2bwtLdAcXFQ1RRPJ2ckWMakwO0nwY='} 2024-04-02T04:35:36.005 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-02T04:35:36.011 INFO:teuthology.task.internal:roles: ubuntu@smithi086.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-02T04:35:36.011 INFO:teuthology.task.internal:roles: ubuntu@smithi192.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-02T04:35:36.012 INFO:teuthology.run_tasks:Running task console_log... 2024-04-02T04:35:36.088 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7ff92df9a0d0>, signals=[15]) 2024-04-02T04:35:36.088 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-02T04:35:36.095 INFO:teuthology.task.internal:Opening connections... 2024-04-02T04:35:36.095 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi086.front.sepia.ceph.com 2024-04-02T04:35:36.097 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi086.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T04:35:36.188 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi192.front.sepia.ceph.com 2024-04-02T04:35:36.188 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi192.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T04:35:36.277 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-02T04:35:36.284 DEBUG:teuthology.orchestra.run.smithi086:> uname -m 2024-04-02T04:35:36.303 INFO:teuthology.orchestra.run.smithi086.stdout:x86_64 2024-04-02T04:35:36.303 DEBUG:teuthology.orchestra.run.smithi086:> cat /etc/os-release 2024-04-02T04:35:36.361 INFO:teuthology.orchestra.run.smithi086.stdout:NAME="CentOS Stream" 2024-04-02T04:35:36.361 INFO:teuthology.orchestra.run.smithi086.stdout:VERSION="9" 2024-04-02T04:35:36.361 INFO:teuthology.orchestra.run.smithi086.stdout:ID="centos" 2024-04-02T04:35:36.361 INFO:teuthology.orchestra.run.smithi086.stdout:ID_LIKE="rhel fedora" 2024-04-02T04:35:36.361 INFO:teuthology.orchestra.run.smithi086.stdout:VERSION_ID="9" 2024-04-02T04:35:36.361 INFO:teuthology.orchestra.run.smithi086.stdout:PLATFORM_ID="platform:el9" 2024-04-02T04:35:36.361 INFO:teuthology.orchestra.run.smithi086.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-02T04:35:36.361 INFO:teuthology.orchestra.run.smithi086.stdout:ANSI_COLOR="0;31" 2024-04-02T04:35:36.361 INFO:teuthology.orchestra.run.smithi086.stdout:LOGO="fedora-logo-icon" 2024-04-02T04:35:36.361 INFO:teuthology.orchestra.run.smithi086.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-02T04:35:36.362 INFO:teuthology.orchestra.run.smithi086.stdout:HOME_URL="https://centos.org/" 2024-04-02T04:35:36.362 INFO:teuthology.orchestra.run.smithi086.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-02T04:35:36.362 INFO:teuthology.orchestra.run.smithi086.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-02T04:35:36.362 INFO:teuthology.orchestra.run.smithi086.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-02T04:35:36.362 INFO:teuthology.lock.ops:Updating smithi086.front.sepia.ceph.com on lock server 2024-04-02T04:35:36.391 DEBUG:teuthology.orchestra.run.smithi192:> uname -m 2024-04-02T04:35:36.407 INFO:teuthology.orchestra.run.smithi192.stdout:x86_64 2024-04-02T04:35:36.408 DEBUG:teuthology.orchestra.run.smithi192:> cat /etc/os-release 2024-04-02T04:35:36.464 INFO:teuthology.orchestra.run.smithi192.stdout:NAME="CentOS Stream" 2024-04-02T04:35:36.464 INFO:teuthology.orchestra.run.smithi192.stdout:VERSION="9" 2024-04-02T04:35:36.464 INFO:teuthology.orchestra.run.smithi192.stdout:ID="centos" 2024-04-02T04:35:36.464 INFO:teuthology.orchestra.run.smithi192.stdout:ID_LIKE="rhel fedora" 2024-04-02T04:35:36.465 INFO:teuthology.orchestra.run.smithi192.stdout:VERSION_ID="9" 2024-04-02T04:35:36.465 INFO:teuthology.orchestra.run.smithi192.stdout:PLATFORM_ID="platform:el9" 2024-04-02T04:35:36.465 INFO:teuthology.orchestra.run.smithi192.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-02T04:35:36.465 INFO:teuthology.orchestra.run.smithi192.stdout:ANSI_COLOR="0;31" 2024-04-02T04:35:36.465 INFO:teuthology.orchestra.run.smithi192.stdout:LOGO="fedora-logo-icon" 2024-04-02T04:35:36.465 INFO:teuthology.orchestra.run.smithi192.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-02T04:35:36.465 INFO:teuthology.orchestra.run.smithi192.stdout:HOME_URL="https://centos.org/" 2024-04-02T04:35:36.465 INFO:teuthology.orchestra.run.smithi192.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-02T04:35:36.465 INFO:teuthology.orchestra.run.smithi192.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-02T04:35:36.465 INFO:teuthology.orchestra.run.smithi192.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-02T04:35:36.465 INFO:teuthology.lock.ops:Updating smithi192.front.sepia.ceph.com on lock server 2024-04-02T04:35:36.490 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-02T04:35:36.500 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-02T04:35:36.505 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-02T04:35:36.505 DEBUG:teuthology.orchestra.run.smithi086:> test '!' -e /home/ubuntu/cephtest 2024-04-02T04:35:36.508 DEBUG:teuthology.orchestra.run.smithi192:> test '!' -e /home/ubuntu/cephtest 2024-04-02T04:35:36.523 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-02T04:35:36.529 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-02T04:35:36.530 DEBUG:teuthology.orchestra.run.smithi086:> test -z $(ls -A /var/lib/ceph) 2024-04-02T04:35:36.565 DEBUG:teuthology.orchestra.run.smithi192:> test -z $(ls -A /var/lib/ceph) 2024-04-02T04:35:36.660 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-02T04:35:36.713 INFO:teuthology.run_tasks:Running task kernel... 2024-04-02T04:35:36.727 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2024-04-02T04:35:36.727 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2024-04-02T04:35:36.728 DEBUG:teuthology.orchestra.run.smithi086:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-02T04:35:36.728 DEBUG:teuthology.orchestra.run.smithi192:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-02T04:35:36.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T04:35:36.744 DEBUG:teuthology.orchestra.run.smithi192:> uname -r 2024-04-02T04:35:36.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T04:35:36.745 DEBUG:teuthology.orchestra.run.smithi086:> uname -r 2024-04-02T04:35:36.800 INFO:teuthology.orchestra.run.smithi192.stdout:5.14.0-361.el9.x86_64 2024-04-02T04:35:36.800 INFO:teuthology.task.kernel:Running kernel on smithi192: 5.14.0-361.el9.x86_64 2024-04-02T04:35:36.800 DEBUG:teuthology.orchestra.run.smithi192:> sudo yum install -y kernel 2024-04-02T04:35:36.804 INFO:teuthology.orchestra.run.smithi086.stdout:5.14.0-361.el9.x86_64 2024-04-02T04:35:36.804 INFO:teuthology.task.kernel:Running kernel on smithi086: 5.14.0-361.el9.x86_64 2024-04-02T04:35:36.804 DEBUG:teuthology.orchestra.run.smithi086:> sudo yum install -y kernel 2024-04-02T04:35:39.020 INFO:teuthology.orchestra.run.smithi086.stdout:CentOS-9 - AppStream 12 MB/s | 6.7 MB 00:00 2024-04-02T04:35:39.166 INFO:teuthology.orchestra.run.smithi192.stdout:CentOS-9 - AppStream 9.6 MB/s | 6.7 MB 00:00 2024-04-02T04:35:40.533 INFO:teuthology.orchestra.run.smithi192.stdout:CentOS-9 - BaseOS 7.4 MB/s | 2.0 MB 00:00 2024-04-02T04:35:40.678 INFO:teuthology.orchestra.run.smithi086.stdout:CentOS-9 - BaseOS 3.6 MB/s | 2.0 MB 00:00 2024-04-02T04:35:41.561 INFO:teuthology.orchestra.run.smithi086.stdout:CentOS Stream 9 - BaseOS 15 MB/s | 8.0 MB 00:00 2024-04-02T04:35:42.282 INFO:teuthology.orchestra.run.smithi192.stdout:CentOS Stream 9 - BaseOS 5.7 MB/s | 8.0 MB 00:01 2024-04-02T04:35:43.371 INFO:teuthology.orchestra.run.smithi086.stdout:CentOS Stream 9 - AppStream 25 MB/s | 19 MB 00:00 2024-04-02T04:35:47.207 INFO:teuthology.orchestra.run.smithi192.stdout:CentOS Stream 9 - AppStream 4.9 MB/s | 19 MB 00:03 2024-04-02T04:35:47.255 INFO:teuthology.orchestra.run.smithi086.stdout:CentOS Stream 9 - CRB 15 MB/s | 6.1 MB 00:00 2024-04-02T04:35:48.689 INFO:teuthology.orchestra.run.smithi086.stdout:CentOS Stream 9 - Extras packages 52 kB/s | 16 kB 00:00 2024-04-02T04:35:50.424 INFO:teuthology.orchestra.run.smithi086.stdout:Extra Packages for Enterprise Linux 13 MB/s | 21 MB 00:01 2024-04-02T04:35:52.550 INFO:teuthology.orchestra.run.smithi192.stdout:CentOS Stream 9 - CRB 3.4 MB/s | 6.1 MB 00:01 2024-04-02T04:35:54.017 INFO:teuthology.orchestra.run.smithi192.stdout:CentOS Stream 9 - Extras packages 48 kB/s | 16 kB 00:00 2024-04-02T04:35:55.340 INFO:teuthology.orchestra.run.smithi086.stdout:lab-extras 16 kB/s | 1.7 kB 00:00 2024-04-02T04:35:56.325 INFO:teuthology.orchestra.run.smithi192.stdout:Extra Packages for Enterprise Linux 9.5 MB/s | 21 MB 00:02 2024-04-02T04:35:57.505 INFO:teuthology.orchestra.run.smithi086.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-02T04:35:57.505 INFO:teuthology.orchestra.run.smithi086.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-04-02T04:35:57.603 INFO:teuthology.orchestra.run.smithi086.stdout:Dependencies resolved. 2024-04-02T04:35:57.609 INFO:teuthology.orchestra.run.smithi086.stdout:================================================================================ 2024-04-02T04:35:57.609 INFO:teuthology.orchestra.run.smithi086.stdout: Package Arch Version Repository Size 2024-04-02T04:35:57.609 INFO:teuthology.orchestra.run.smithi086.stdout:================================================================================ 2024-04-02T04:35:57.609 INFO:teuthology.orchestra.run.smithi086.stdout:Installing: 2024-04-02T04:35:57.609 INFO:teuthology.orchestra.run.smithi086.stdout: kernel x86_64 5.14.0-434.el9 CentOS-BaseOS 192 k 2024-04-02T04:35:57.609 INFO:teuthology.orchestra.run.smithi086.stdout:Installing dependencies: 2024-04-02T04:35:57.609 INFO:teuthology.orchestra.run.smithi086.stdout: kernel-core x86_64 5.14.0-434.el9 CentOS-BaseOS 15 M 2024-04-02T04:35:57.609 INFO:teuthology.orchestra.run.smithi086.stdout: kernel-modules x86_64 5.14.0-434.el9 CentOS-BaseOS 34 M 2024-04-02T04:35:57.610 INFO:teuthology.orchestra.run.smithi086.stdout: kernel-modules-core x86_64 5.14.0-434.el9 CentOS-BaseOS 28 M 2024-04-02T04:35:57.610 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-02T04:35:57.610 INFO:teuthology.orchestra.run.smithi086.stdout:Transaction Summary 2024-04-02T04:35:57.610 INFO:teuthology.orchestra.run.smithi086.stdout:================================================================================ 2024-04-02T04:35:57.610 INFO:teuthology.orchestra.run.smithi086.stdout:Install 4 Packages 2024-04-02T04:35:57.610 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-02T04:35:57.610 INFO:teuthology.orchestra.run.smithi086.stdout:Total download size: 76 M 2024-04-02T04:35:57.611 INFO:teuthology.orchestra.run.smithi086.stdout:Installed size: 123 M 2024-04-02T04:35:57.611 INFO:teuthology.orchestra.run.smithi086.stdout:Downloading Packages: 2024-04-02T04:35:57.766 INFO:teuthology.orchestra.run.smithi086.stdout:(1/4): kernel-5.14.0-434.el9.x86_64.rpm 1.2 MB/s | 192 kB 00:00 2024-04-02T04:35:58.325 INFO:teuthology.orchestra.run.smithi086.stdout:(2/4): kernel-core-5.14.0-434.el9.x86_64.rpm 21 MB/s | 15 MB 00:00 2024-04-02T04:35:58.934 INFO:teuthology.orchestra.run.smithi086.stdout:(3/4): kernel-modules-core-5.14.0-434.el9.x86_6 24 MB/s | 28 MB 00:01 2024-04-02T04:35:59.226 INFO:teuthology.orchestra.run.smithi086.stdout:(4/4): kernel-modules-5.14.0-434.el9.x86_64.rpm 21 MB/s | 34 MB 00:01 2024-04-02T04:35:59.228 INFO:teuthology.orchestra.run.smithi086.stdout:-------------------------------------------------------------------------------- 2024-04-02T04:35:59.229 INFO:teuthology.orchestra.run.smithi086.stdout:Total 47 MB/s | 76 MB 00:01 2024-04-02T04:35:59.229 INFO:teuthology.orchestra.run.smithi086.stdout:Running transaction check 2024-04-02T04:35:59.570 INFO:teuthology.orchestra.run.smithi086.stdout:Transaction check succeeded. 2024-04-02T04:35:59.570 INFO:teuthology.orchestra.run.smithi086.stdout:Running transaction test 2024-04-02T04:36:00.419 INFO:teuthology.orchestra.run.smithi086.stdout:Transaction test succeeded. 2024-04-02T04:36:00.419 INFO:teuthology.orchestra.run.smithi086.stdout:Running transaction 2024-04-02T04:36:01.422 INFO:teuthology.orchestra.run.smithi192.stdout:lab-extras 12 kB/s | 1.7 kB 00:00 2024-04-02T04:36:02.720 INFO:teuthology.orchestra.run.smithi086.stdout: Preparing : 1/1 2024-04-02T04:36:03.297 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : kernel-modules-core-5.14.0-434.el9.x86_64 1/4 2024-04-02T04:36:03.384 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : kernel-core-5.14.0-434.el9.x86_64 2/4 2024-04-02T04:36:03.725 INFO:teuthology.orchestra.run.smithi192.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-02T04:36:03.725 INFO:teuthology.orchestra.run.smithi192.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-04-02T04:36:03.810 INFO:teuthology.orchestra.run.smithi192.stdout:Dependencies resolved. 2024-04-02T04:36:03.823 INFO:teuthology.orchestra.run.smithi192.stdout:================================================================================ 2024-04-02T04:36:03.823 INFO:teuthology.orchestra.run.smithi192.stdout: Package Arch Version Repository Size 2024-04-02T04:36:03.823 INFO:teuthology.orchestra.run.smithi192.stdout:================================================================================ 2024-04-02T04:36:03.823 INFO:teuthology.orchestra.run.smithi192.stdout:Installing: 2024-04-02T04:36:03.823 INFO:teuthology.orchestra.run.smithi192.stdout: kernel x86_64 5.14.0-434.el9 CentOS-BaseOS 192 k 2024-04-02T04:36:03.823 INFO:teuthology.orchestra.run.smithi192.stdout:Installing dependencies: 2024-04-02T04:36:03.823 INFO:teuthology.orchestra.run.smithi192.stdout: kernel-core x86_64 5.14.0-434.el9 CentOS-BaseOS 15 M 2024-04-02T04:36:03.823 INFO:teuthology.orchestra.run.smithi192.stdout: kernel-modules x86_64 5.14.0-434.el9 CentOS-BaseOS 34 M 2024-04-02T04:36:03.823 INFO:teuthology.orchestra.run.smithi192.stdout: kernel-modules-core x86_64 5.14.0-434.el9 CentOS-BaseOS 28 M 2024-04-02T04:36:03.823 INFO:teuthology.orchestra.run.smithi192.stdout: 2024-04-02T04:36:03.823 INFO:teuthology.orchestra.run.smithi192.stdout:Transaction Summary 2024-04-02T04:36:03.824 INFO:teuthology.orchestra.run.smithi192.stdout:================================================================================ 2024-04-02T04:36:03.824 INFO:teuthology.orchestra.run.smithi192.stdout:Install 4 Packages 2024-04-02T04:36:03.824 INFO:teuthology.orchestra.run.smithi192.stdout: 2024-04-02T04:36:03.824 INFO:teuthology.orchestra.run.smithi192.stdout:Total download size: 76 M 2024-04-02T04:36:03.824 INFO:teuthology.orchestra.run.smithi192.stdout:Installed size: 123 M 2024-04-02T04:36:03.824 INFO:teuthology.orchestra.run.smithi192.stdout:Downloading Packages: 2024-04-02T04:36:03.970 INFO:teuthology.orchestra.run.smithi192.stdout:(1/4): kernel-5.14.0-434.el9.x86_64.rpm 1.3 MB/s | 192 kB 00:00 2024-04-02T04:36:04.395 INFO:teuthology.orchestra.run.smithi192.stdout:(2/4): kernel-core-5.14.0-434.el9.x86_64.rpm 26 MB/s | 15 MB 00:00 2024-04-02T04:36:05.045 INFO:teuthology.orchestra.run.smithi192.stdout:(3/4): kernel-modules-5.14.0-434.el9.x86_64.rpm 28 MB/s | 34 MB 00:01 2024-04-02T04:36:05.520 INFO:teuthology.orchestra.run.smithi192.stdout:(4/4): kernel-modules-core-5.14.0-434.el9.x86_6 18 MB/s | 28 MB 00:01 2024-04-02T04:36:05.523 INFO:teuthology.orchestra.run.smithi192.stdout:-------------------------------------------------------------------------------- 2024-04-02T04:36:05.523 INFO:teuthology.orchestra.run.smithi192.stdout:Total 45 MB/s | 76 MB 00:01 2024-04-02T04:36:05.523 INFO:teuthology.orchestra.run.smithi192.stdout:Running transaction check 2024-04-02T04:36:05.887 INFO:teuthology.orchestra.run.smithi192.stdout:Transaction check succeeded. 2024-04-02T04:36:05.887 INFO:teuthology.orchestra.run.smithi192.stdout:Running transaction test 2024-04-02T04:36:06.155 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: kernel-core-5.14.0-434.el9.x86_64 2/4 2024-04-02T04:36:06.285 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-04-02T04:36:06.727 INFO:teuthology.orchestra.run.smithi192.stdout:Transaction test succeeded. 2024-04-02T04:36:06.727 INFO:teuthology.orchestra.run.smithi192.stdout:Running transaction 2024-04-02T04:36:08.989 INFO:teuthology.orchestra.run.smithi192.stdout: Preparing : 1/1 2024-04-02T04:36:09.623 INFO:teuthology.orchestra.run.smithi192.stdout: Installing : kernel-modules-core-5.14.0-434.el9.x86_64 1/4 2024-04-02T04:36:09.706 INFO:teuthology.orchestra.run.smithi192.stdout: Installing : kernel-core-5.14.0-434.el9.x86_64 2/4 2024-04-02T04:36:11.024 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-04-02T04:36:11.058 INFO:teuthology.orchestra.run.smithi086.stdout: Installing : kernel-5.14.0-434.el9.x86_64 4/4 2024-04-02T04:36:12.468 INFO:teuthology.orchestra.run.smithi192.stdout: Running scriptlet: kernel-core-5.14.0-434.el9.x86_64 2/4 2024-04-02T04:36:12.608 INFO:teuthology.orchestra.run.smithi192.stdout: Installing : kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-04-02T04:36:15.778 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: kernel-modules-core-5.14.0-434.el9.x86_64 4/4 2024-04-02T04:36:17.363 INFO:teuthology.orchestra.run.smithi192.stdout: Running scriptlet: kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-04-02T04:36:17.482 INFO:teuthology.orchestra.run.smithi192.stdout: Installing : kernel-5.14.0-434.el9.x86_64 4/4 2024-04-02T04:36:22.283 INFO:teuthology.orchestra.run.smithi192.stdout: Running scriptlet: kernel-modules-core-5.14.0-434.el9.x86_64 4/4 2024-04-02T04:36:45.259 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: kernel-core-5.14.0-434.el9.x86_64 4/4 2024-04-02T04:36:45.259 INFO:teuthology.orchestra.run.smithi086.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-04-02T04:36:45.259 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-02T04:36:45.301 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: kernel-modules-5.14.0-434.el9.x86_64 4/4 2024-04-02T04:36:45.633 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: kernel-5.14.0-434.el9.x86_64 4/4 2024-04-02T04:36:45.634 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : kernel-5.14.0-434.el9.x86_64 1/4 2024-04-02T04:36:45.634 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : kernel-core-5.14.0-434.el9.x86_64 2/4 2024-04-02T04:36:45.634 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-04-02T04:36:46.038 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : kernel-modules-core-5.14.0-434.el9.x86_64 4/4 2024-04-02T04:36:46.038 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-02T04:36:46.038 INFO:teuthology.orchestra.run.smithi086.stdout:Installed: 2024-04-02T04:36:46.038 INFO:teuthology.orchestra.run.smithi086.stdout: kernel-5.14.0-434.el9.x86_64 2024-04-02T04:36:46.038 INFO:teuthology.orchestra.run.smithi086.stdout: kernel-core-5.14.0-434.el9.x86_64 2024-04-02T04:36:46.038 INFO:teuthology.orchestra.run.smithi086.stdout: kernel-modules-5.14.0-434.el9.x86_64 2024-04-02T04:36:46.038 INFO:teuthology.orchestra.run.smithi086.stdout: kernel-modules-core-5.14.0-434.el9.x86_64 2024-04-02T04:36:46.038 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-02T04:36:46.038 INFO:teuthology.orchestra.run.smithi086.stdout:Complete! 2024-04-02T04:36:46.401 DEBUG:teuthology.orchestra.run.smithi086:> rpm -q kernel | sort -rV | head -n 1 2024-04-02T04:36:46.498 INFO:teuthology.orchestra.run.smithi086.stdout:kernel-5.14.0-434.el9.x86_64 2024-04-02T04:36:46.499 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-434.el9.x86_64 2024-04-02T04:36:46.499 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-434.el9.x86_64 2024-04-02T04:36:46.499 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-02T04:36:46.499 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-02T04:36:46.499 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-04-02T04:36:46.499 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi086.front.sepia.ceph.com, path=None, version=distro) 2024-04-02T04:36:46.500 DEBUG:teuthology.orchestra.run.smithi086:> rpm -q kernel | sort -rV | head -n 1 2024-04-02T04:36:46.579 INFO:teuthology.orchestra.run.smithi086.stdout:kernel-5.14.0-434.el9.x86_64 2024-04-02T04:36:46.579 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-434.el9.x86_64 2024-04-02T04:36:46.579 DEBUG:teuthology.orchestra.run.smithi086:> sudo rpm -qi grub2-tools 2024-04-02T04:36:46.619 INFO:teuthology.orchestra.run.smithi086.stdout:Name : grub2-tools 2024-04-02T04:36:46.619 INFO:teuthology.orchestra.run.smithi086.stdout:Epoch : 1 2024-04-02T04:36:46.619 INFO:teuthology.orchestra.run.smithi086.stdout:Version : 2.06 2024-04-02T04:36:46.619 INFO:teuthology.orchestra.run.smithi086.stdout:Release : 68.el9 2024-04-02T04:36:46.619 INFO:teuthology.orchestra.run.smithi086.stdout:Architecture: x86_64 2024-04-02T04:36:46.619 INFO:teuthology.orchestra.run.smithi086.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-04-02T04:36:46.619 INFO:teuthology.orchestra.run.smithi086.stdout:Group : Unspecified 2024-04-02T04:36:46.619 INFO:teuthology.orchestra.run.smithi086.stdout:Size : 8264254 2024-04-02T04:36:46.619 INFO:teuthology.orchestra.run.smithi086.stdout:License : GPLv3+ 2024-04-02T04:36:46.619 INFO:teuthology.orchestra.run.smithi086.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-04-02T04:36:46.619 INFO:teuthology.orchestra.run.smithi086.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-04-02T04:36:46.620 INFO:teuthology.orchestra.run.smithi086.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-04-02T04:36:46.620 INFO:teuthology.orchestra.run.smithi086.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-04-02T04:36:46.620 INFO:teuthology.orchestra.run.smithi086.stdout:Packager : builder@centos.org 2024-04-02T04:36:46.620 INFO:teuthology.orchestra.run.smithi086.stdout:Vendor : CentOS 2024-04-02T04:36:46.620 INFO:teuthology.orchestra.run.smithi086.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-02T04:36:46.620 INFO:teuthology.orchestra.run.smithi086.stdout:Summary : Support tools for GRUB. 2024-04-02T04:36:46.620 INFO:teuthology.orchestra.run.smithi086.stdout:Description : 2024-04-02T04:36:46.620 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-02T04:36:46.620 INFO:teuthology.orchestra.run.smithi086.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-02T04:36:46.620 INFO:teuthology.orchestra.run.smithi086.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-02T04:36:46.620 INFO:teuthology.orchestra.run.smithi086.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-02T04:36:46.620 INFO:teuthology.orchestra.run.smithi086.stdout:hardware devices. 2024-04-02T04:36:46.620 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-02T04:36:46.620 INFO:teuthology.orchestra.run.smithi086.stdout:This subpackage provides tools for support of all platforms. 2024-04-02T04:36:46.621 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-02T04:36:46.621 INFO:teuthology.task.kernel:Updating grub on smithi086 to boot 5.14.0-434.el9.x86_64 2024-04-02T04:36:46.622 DEBUG:teuthology.orchestra.run.smithi086:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-02T04:36:47.184 INFO:teuthology.orchestra.run.smithi086.stderr:Generating grub configuration file ... 2024-04-02T04:36:48.562 INFO:teuthology.orchestra.run.smithi086.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-04-02T04:36:48.604 INFO:teuthology.orchestra.run.smithi086.stderr:done 2024-04-02T04:36:48.607 DEBUG:teuthology.orchestra.run.smithi086:> mktemp 2024-04-02T04:36:48.622 INFO:teuthology.orchestra.run.smithi086.stdout:/tmp/tmp.M39kVOTenl 2024-04-02T04:36:48.623 DEBUG:teuthology.orchestra.run.smithi086:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.M39kVOTenl 2024-04-02T04:36:48.691 DEBUG:teuthology.orchestra.run.smithi086:> sudo chmod 0666 /tmp/tmp.M39kVOTenl 2024-04-02T04:36:48.826 DEBUG:teuthology.orchestra.remote:smithi086:/tmp/tmp.M39kVOTenl is 6KB 2024-04-02T04:36:48.875 DEBUG:teuthology.orchestra.run.smithi086:> rm -fr /tmp/tmp.M39kVOTenl 2024-04-02T04:36:48.889 DEBUG:teuthology.orchestra.run.smithi086:> sudo /bin/ls /boot/loader/entries || true 2024-04-02T04:36:48.959 INFO:teuthology.orchestra.run.smithi086.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-04-02T04:36:48.959 INFO:teuthology.orchestra.run.smithi086.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-04-02T04:36:48.959 INFO:teuthology.orchestra.run.smithi086.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-04-02T04:36:48.959 INFO:teuthology.orchestra.run.smithi086.stdout:15949e3c88704300882482eddfd7baae-5.14.0-434.el9.x86_64.conf 2024-04-02T04:36:48.960 DEBUG:teuthology.orchestra.run.smithi086:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-434.el9.x86_64 2024-04-02T04:36:49.126 DEBUG:teuthology.orchestra.run.smithi086:> sudo shutdown -r now 2024-04-02T04:36:51.864 INFO:teuthology.orchestra.run.smithi192.stdout: Running scriptlet: kernel-core-5.14.0-434.el9.x86_64 4/4 2024-04-02T04:36:51.864 INFO:teuthology.orchestra.run.smithi192.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-04-02T04:36:51.864 INFO:teuthology.orchestra.run.smithi192.stdout: 2024-04-02T04:36:51.908 INFO:teuthology.orchestra.run.smithi192.stdout: Running scriptlet: kernel-modules-5.14.0-434.el9.x86_64 4/4 2024-04-02T04:36:52.241 INFO:teuthology.orchestra.run.smithi192.stdout: Running scriptlet: kernel-5.14.0-434.el9.x86_64 4/4 2024-04-02T04:36:52.241 INFO:teuthology.orchestra.run.smithi192.stdout: Verifying : kernel-5.14.0-434.el9.x86_64 1/4 2024-04-02T04:36:52.241 INFO:teuthology.orchestra.run.smithi192.stdout: Verifying : kernel-core-5.14.0-434.el9.x86_64 2/4 2024-04-02T04:36:52.241 INFO:teuthology.orchestra.run.smithi192.stdout: Verifying : kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-04-02T04:36:52.608 INFO:teuthology.orchestra.run.smithi192.stdout: Verifying : kernel-modules-core-5.14.0-434.el9.x86_64 4/4 2024-04-02T04:36:52.608 INFO:teuthology.orchestra.run.smithi192.stdout: 2024-04-02T04:36:52.608 INFO:teuthology.orchestra.run.smithi192.stdout:Installed: 2024-04-02T04:36:52.608 INFO:teuthology.orchestra.run.smithi192.stdout: kernel-5.14.0-434.el9.x86_64 2024-04-02T04:36:52.609 INFO:teuthology.orchestra.run.smithi192.stdout: kernel-core-5.14.0-434.el9.x86_64 2024-04-02T04:36:52.609 INFO:teuthology.orchestra.run.smithi192.stdout: kernel-modules-5.14.0-434.el9.x86_64 2024-04-02T04:36:52.609 INFO:teuthology.orchestra.run.smithi192.stdout: kernel-modules-core-5.14.0-434.el9.x86_64 2024-04-02T04:36:52.609 INFO:teuthology.orchestra.run.smithi192.stdout: 2024-04-02T04:36:52.609 INFO:teuthology.orchestra.run.smithi192.stdout:Complete! 2024-04-02T04:36:52.951 DEBUG:teuthology.orchestra.run.smithi192:> rpm -q kernel | sort -rV | head -n 1 2024-04-02T04:36:53.051 INFO:teuthology.orchestra.run.smithi192.stdout:kernel-5.14.0-434.el9.x86_64 2024-04-02T04:36:53.052 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-434.el9.x86_64 2024-04-02T04:36:53.052 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-434.el9.x86_64 2024-04-02T04:36:53.052 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-02T04:36:53.052 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-02T04:36:53.052 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-04-02T04:36:53.052 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi192.front.sepia.ceph.com, path=None, version=distro) 2024-04-02T04:36:53.052 DEBUG:teuthology.orchestra.run.smithi192:> rpm -q kernel | sort -rV | head -n 1 2024-04-02T04:36:53.133 INFO:teuthology.orchestra.run.smithi192.stdout:kernel-5.14.0-434.el9.x86_64 2024-04-02T04:36:53.133 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-434.el9.x86_64 2024-04-02T04:36:53.133 DEBUG:teuthology.orchestra.run.smithi192:> sudo rpm -qi grub2-tools 2024-04-02T04:36:53.177 INFO:teuthology.orchestra.run.smithi192.stdout:Name : grub2-tools 2024-04-02T04:36:53.178 INFO:teuthology.orchestra.run.smithi192.stdout:Epoch : 1 2024-04-02T04:36:53.178 INFO:teuthology.orchestra.run.smithi192.stdout:Version : 2.06 2024-04-02T04:36:53.178 INFO:teuthology.orchestra.run.smithi192.stdout:Release : 68.el9 2024-04-02T04:36:53.178 INFO:teuthology.orchestra.run.smithi192.stdout:Architecture: x86_64 2024-04-02T04:36:53.178 INFO:teuthology.orchestra.run.smithi192.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-04-02T04:36:53.178 INFO:teuthology.orchestra.run.smithi192.stdout:Group : Unspecified 2024-04-02T04:36:53.178 INFO:teuthology.orchestra.run.smithi192.stdout:Size : 8264254 2024-04-02T04:36:53.178 INFO:teuthology.orchestra.run.smithi192.stdout:License : GPLv3+ 2024-04-02T04:36:53.178 INFO:teuthology.orchestra.run.smithi192.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-04-02T04:36:53.178 INFO:teuthology.orchestra.run.smithi192.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-04-02T04:36:53.178 INFO:teuthology.orchestra.run.smithi192.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-04-02T04:36:53.178 INFO:teuthology.orchestra.run.smithi192.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-04-02T04:36:53.178 INFO:teuthology.orchestra.run.smithi192.stdout:Packager : builder@centos.org 2024-04-02T04:36:53.179 INFO:teuthology.orchestra.run.smithi192.stdout:Vendor : CentOS 2024-04-02T04:36:53.179 INFO:teuthology.orchestra.run.smithi192.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-02T04:36:53.179 INFO:teuthology.orchestra.run.smithi192.stdout:Summary : Support tools for GRUB. 2024-04-02T04:36:53.179 INFO:teuthology.orchestra.run.smithi192.stdout:Description : 2024-04-02T04:36:53.179 INFO:teuthology.orchestra.run.smithi192.stdout: 2024-04-02T04:36:53.179 INFO:teuthology.orchestra.run.smithi192.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-02T04:36:53.179 INFO:teuthology.orchestra.run.smithi192.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-02T04:36:53.179 INFO:teuthology.orchestra.run.smithi192.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-02T04:36:53.179 INFO:teuthology.orchestra.run.smithi192.stdout:hardware devices. 2024-04-02T04:36:53.179 INFO:teuthology.orchestra.run.smithi192.stdout: 2024-04-02T04:36:53.179 INFO:teuthology.orchestra.run.smithi192.stdout:This subpackage provides tools for support of all platforms. 2024-04-02T04:36:53.180 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-02T04:36:53.180 INFO:teuthology.task.kernel:Updating grub on smithi192 to boot 5.14.0-434.el9.x86_64 2024-04-02T04:36:53.180 DEBUG:teuthology.orchestra.run.smithi192:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-02T04:36:53.739 INFO:teuthology.orchestra.run.smithi192.stderr:Generating grub configuration file ... 2024-04-02T04:36:55.019 INFO:teuthology.orchestra.run.smithi192.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-04-02T04:36:55.054 INFO:teuthology.orchestra.run.smithi192.stderr:done 2024-04-02T04:36:55.056 DEBUG:teuthology.orchestra.run.smithi192:> mktemp 2024-04-02T04:36:55.071 INFO:teuthology.orchestra.run.smithi192.stdout:/tmp/tmp.SHb9CeIMBO 2024-04-02T04:36:55.072 DEBUG:teuthology.orchestra.run.smithi192:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.SHb9CeIMBO 2024-04-02T04:36:55.140 DEBUG:teuthology.orchestra.run.smithi192:> sudo chmod 0666 /tmp/tmp.SHb9CeIMBO 2024-04-02T04:36:55.281 DEBUG:teuthology.orchestra.remote:smithi192:/tmp/tmp.SHb9CeIMBO is 6KB 2024-04-02T04:36:55.333 DEBUG:teuthology.orchestra.run.smithi192:> rm -fr /tmp/tmp.SHb9CeIMBO 2024-04-02T04:36:55.347 DEBUG:teuthology.orchestra.run.smithi192:> sudo /bin/ls /boot/loader/entries || true 2024-04-02T04:36:55.411 INFO:teuthology.orchestra.run.smithi192.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-04-02T04:36:55.411 INFO:teuthology.orchestra.run.smithi192.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-04-02T04:36:55.411 INFO:teuthology.orchestra.run.smithi192.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-04-02T04:36:55.411 INFO:teuthology.orchestra.run.smithi192.stdout:15949e3c88704300882482eddfd7baae-5.14.0-434.el9.x86_64.conf 2024-04-02T04:36:55.413 DEBUG:teuthology.orchestra.run.smithi192:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-434.el9.x86_64 2024-04-02T04:36:55.581 DEBUG:teuthology.orchestra.run.smithi192:> sudo shutdown -r now 2024-04-02T04:37:19.150 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-04-02T04:37:19.151 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi086.front.sepia.ceph.com' 2024-04-02T04:37:19.152 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi086.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T04:37:25.591 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-04-02T04:37:25.591 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi192.front.sepia.ceph.com' 2024-04-02T04:37:25.592 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi192.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T04:37:37.694 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.86 2024-04-02T04:37:44.094 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.192 2024-04-02T04:37:46.697 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi086.front.sepia.ceph.com' 2024-04-02T04:37:46.698 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi086.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T04:37:49.758 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.86 2024-04-02T04:37:53.098 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi192.front.sepia.ceph.com' 2024-04-02T04:37:53.099 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi192.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T04:38:01.765 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi086.front.sepia.ceph.com' 2024-04-02T04:38:01.766 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi086.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T04:38:20.190 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.86 2024-04-02T04:38:35.197 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi086.front.sepia.ceph.com' 2024-04-02T04:38:35.198 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi086.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T04:38:35.619 DEBUG:teuthology.orchestra.run.smithi086:> true 2024-04-02T04:38:36.029 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi086.front.sepia.ceph.com' 2024-04-02T04:38:36.029 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-434.el9.x86_64"... 2024-04-02T04:38:36.029 DEBUG:teuthology.orchestra.run.smithi086:> uname -r 2024-04-02T04:38:36.043 INFO:teuthology.orchestra.run.smithi086.stdout:5.14.0-434.el9.x86_64 2024-04-02T04:38:36.043 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-434.el9.x86_64 vs 5.14.0-434.el9.x86_64 2024-04-02T04:38:36.044 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-02T04:38:36.044 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-02T04:38:37.044 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-02T04:38:37.044 DEBUG:teuthology.orchestra.run.smithi086:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-02T04:38:37.103 INFO:teuthology.orchestra.run.smithi086.stdout:ttyS1 2024-04-02T04:38:37.134 DEBUG:teuthology.parallel:result is None 2024-04-02T04:38:53.101 DEBUG:teuthology.orchestra.remote:timed out 2024-04-02T04:39:05.102 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi192.front.sepia.ceph.com' 2024-04-02T04:39:05.104 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi192.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T04:39:05.506 DEBUG:teuthology.orchestra.run.smithi192:> true 2024-04-02T04:39:05.862 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi192.front.sepia.ceph.com' 2024-04-02T04:39:05.862 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-434.el9.x86_64"... 2024-04-02T04:39:05.862 DEBUG:teuthology.orchestra.run.smithi192:> uname -r 2024-04-02T04:39:05.877 INFO:teuthology.orchestra.run.smithi192.stdout:5.14.0-434.el9.x86_64 2024-04-02T04:39:05.877 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-434.el9.x86_64 vs 5.14.0-434.el9.x86_64 2024-04-02T04:39:05.878 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-02T04:39:05.878 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-02T04:39:06.878 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-02T04:39:06.879 DEBUG:teuthology.orchestra.run.smithi192:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-02T04:39:06.944 INFO:teuthology.orchestra.run.smithi192.stdout:ttyS1 2024-04-02T04:39:06.988 DEBUG:teuthology.parallel:result is None 2024-04-02T04:39:06.988 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-02T04:39:06.995 INFO:teuthology.task.internal:Creating test directory... 2024-04-02T04:39:06.996 DEBUG:teuthology.orchestra.run.smithi086:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-02T04:39:06.999 DEBUG:teuthology.orchestra.run.smithi192:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-02T04:39:07.016 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-02T04:39:07.022 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-02T04:39:07.028 INFO:teuthology.task.internal:Creating archive directory... 2024-04-02T04:39:07.028 DEBUG:teuthology.orchestra.run.smithi086:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-02T04:39:07.057 DEBUG:teuthology.orchestra.run.smithi192:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-02T04:39:07.148 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-02T04:39:07.155 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-02T04:39:07.155 DEBUG:teuthology.orchestra.run.smithi086:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-02T04:39:07.157 DEBUG:teuthology.orchestra.run.smithi192:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-02T04:39:07.191 INFO:teuthology.orchestra.run.smithi086.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-02T04:39:07.199 INFO:teuthology.orchestra.run.smithi192.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-02T04:39:07.205 INFO:teuthology.orchestra.run.smithi086.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-02T04:39:07.213 INFO:teuthology.orchestra.run.smithi192.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-02T04:39:07.215 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-02T04:39:07.222 INFO:teuthology.task.internal:Configuring sudo... 2024-04-02T04:39:07.222 DEBUG:teuthology.orchestra.run.smithi086:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-02T04:39:07.250 DEBUG:teuthology.orchestra.run.smithi192:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-02T04:39:07.284 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-02T04:39:07.294 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-02T04:39:07.294 DEBUG:teuthology.orchestra.run.smithi086:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-02T04:39:07.321 DEBUG:teuthology.orchestra.run.smithi192:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-02T04:39:07.341 DEBUG:teuthology.orchestra.run.smithi086:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-02T04:39:07.418 DEBUG:teuthology.orchestra.run.smithi086:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-02T04:39:07.478 DEBUG:teuthology.orchestra.run.smithi086:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-02T04:39:07.519 DEBUG:teuthology.orchestra.run.smithi086:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-02T04:39:07.564 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-04-02T04:39:07.564 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-02T04:39:07.631 DEBUG:teuthology.orchestra.run.smithi192:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-02T04:39:07.675 DEBUG:teuthology.orchestra.run.smithi192:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-02T04:39:07.728 DEBUG:teuthology.orchestra.run.smithi192:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-02T04:39:07.814 DEBUG:teuthology.orchestra.run.smithi192:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-02T04:39:07.861 DEBUG:teuthology.orchestra.run.smithi192:> set -ex 2024-04-02T04:39:07.861 DEBUG:teuthology.orchestra.run.smithi192:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-02T04:39:07.930 DEBUG:teuthology.orchestra.run.smithi086:> sudo service rsyslog restart 2024-04-02T04:39:07.933 DEBUG:teuthology.orchestra.run.smithi192:> sudo service rsyslog restart 2024-04-02T04:39:07.992 INFO:teuthology.orchestra.run.smithi086.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-02T04:39:08.015 INFO:teuthology.orchestra.run.smithi192.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-02T04:39:08.469 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-02T04:39:08.477 INFO:teuthology.task.internal:Starting timer... 2024-04-02T04:39:08.478 INFO:teuthology.run_tasks:Running task pcp... 2024-04-02T04:39:08.538 INFO:teuthology.run_tasks:Running task selinux... 2024-04-02T04:39:08.547 DEBUG:teuthology.orchestra.run.smithi086:> sudo service auditd rotate 2024-04-02T04:39:08.621 INFO:teuthology.orchestra.run.smithi086.stdout:Rotating logs: 2024-04-02T04:39:08.623 DEBUG:teuthology.orchestra.run.smithi192:> sudo service auditd rotate 2024-04-02T04:39:08.688 INFO:teuthology.orchestra.run.smithi192.stdout:Rotating logs: 2024-04-02T04:39:08.690 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-02T04:39:08.690 DEBUG:teuthology.orchestra.run.smithi086:> /usr/sbin/getenforce 2024-04-02T04:39:08.711 INFO:teuthology.orchestra.run.smithi086.stdout:Permissive 2024-04-02T04:39:08.711 DEBUG:teuthology.orchestra.run.smithi192:> /usr/sbin/getenforce 2024-04-02T04:39:08.731 INFO:teuthology.orchestra.run.smithi192.stdout:Permissive 2024-04-02T04:39:08.731 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi086.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi192.front.sepia.ceph.com': 'permissive'} 2024-04-02T04:39:08.731 DEBUG:teuthology.orchestra.run.smithi086:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-02T04:39:08.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T04:39:08.780 DEBUG:teuthology.orchestra.run.smithi192:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-02T04:39:08.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T04:39:08.809 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-02T04:39:08.809 DEBUG:teuthology.orchestra.run.smithi086:> sudo /usr/sbin/setenforce permissive 2024-04-02T04:39:08.855 DEBUG:teuthology.orchestra.run.smithi192:> sudo /usr/sbin/setenforce permissive 2024-04-02T04:39:08.893 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-02T04:39:08.903 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-02T04:39:08.919 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-02T04:39:08.921 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi086.front.sepia.ceph.com,smithi192.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-02T04:43:20.661 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi086.front.sepia.ceph.com'), Remote(name='ubuntu@smithi192.front.sepia.ceph.com')] 2024-04-02T04:43:20.663 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi086.front.sepia.ceph.com' 2024-04-02T04:43:20.665 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi086.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T04:43:20.759 DEBUG:teuthology.orchestra.run.smithi086:> true 2024-04-02T04:43:20.837 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi086.front.sepia.ceph.com' 2024-04-02T04:43:20.838 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi192.front.sepia.ceph.com' 2024-04-02T04:43:20.838 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi192.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T04:43:20.927 DEBUG:teuthology.orchestra.run.smithi192:> true 2024-04-02T04:43:21.001 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi192.front.sepia.ceph.com' 2024-04-02T04:43:21.002 INFO:teuthology.run_tasks:Running task clock... 2024-04-02T04:43:21.014 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-04-02T04:43:21.014 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-02T04:43:21.015 DEBUG:teuthology.orchestra.run.smithi086:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-02T04:43:21.017 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-02T04:43:21.018 DEBUG:teuthology.orchestra.run.smithi192:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-02T04:43:21.045 INFO:teuthology.orchestra.run.smithi086.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-02T04:43:21.062 INFO:teuthology.orchestra.run.smithi086.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-02T04:43:21.072 INFO:teuthology.orchestra.run.smithi192.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-02T04:43:21.089 INFO:teuthology.orchestra.run.smithi192.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-02T04:43:21.091 INFO:teuthology.orchestra.run.smithi086.stderr:sudo: ntpd: command not found 2024-04-02T04:43:21.105 INFO:teuthology.orchestra.run.smithi086.stdout:506 Cannot talk to daemon 2024-04-02T04:43:21.118 INFO:teuthology.orchestra.run.smithi192.stderr:sudo: ntpd: command not found 2024-04-02T04:43:21.121 INFO:teuthology.orchestra.run.smithi086.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-02T04:43:21.132 INFO:teuthology.orchestra.run.smithi192.stdout:506 Cannot talk to daemon 2024-04-02T04:43:21.138 INFO:teuthology.orchestra.run.smithi086.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-02T04:43:21.148 INFO:teuthology.orchestra.run.smithi192.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-02T04:43:21.165 INFO:teuthology.orchestra.run.smithi192.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-02T04:43:21.195 INFO:teuthology.orchestra.run.smithi086.stderr:bash: line 1: ntpq: command not found 2024-04-02T04:43:21.198 INFO:teuthology.orchestra.run.smithi086.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-02T04:43:21.198 INFO:teuthology.orchestra.run.smithi086.stdout:=============================================================================== 2024-04-02T04:43:21.198 INFO:teuthology.orchestra.run.smithi086.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T04:43:21.198 INFO:teuthology.orchestra.run.smithi086.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T04:43:21.199 INFO:teuthology.orchestra.run.smithi086.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T04:43:21.199 INFO:teuthology.orchestra.run.smithi086.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T04:43:21.215 INFO:teuthology.orchestra.run.smithi192.stderr:bash: line 1: ntpq: command not found 2024-04-02T04:43:21.218 INFO:teuthology.orchestra.run.smithi192.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-02T04:43:21.218 INFO:teuthology.orchestra.run.smithi192.stdout:=============================================================================== 2024-04-02T04:43:21.218 INFO:teuthology.orchestra.run.smithi192.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T04:43:21.218 INFO:teuthology.orchestra.run.smithi192.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T04:43:21.219 INFO:teuthology.orchestra.run.smithi192.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T04:43:21.219 INFO:teuthology.orchestra.run.smithi192.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T04:43:21.220 INFO:teuthology.run_tasks:Running task cephadm... 2024-04-02T04:43:21.331 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1, 'mgr/cephadm/use_agent': False}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '6cec793d9073f0594e2364b9b687ecaf22424cca'} 2024-04-02T04:43:21.331 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-04-02T04:43:21.332 INFO:tasks.cephadm:Cluster fsid is 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 2024-04-02T04:43:21.332 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-04-02T04:43:21.332 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.86', 'mon.c': '[v2:172.21.15.86:3301,v1:172.21.15.86:6790]', 'mon.b': '172.21.15.192'} 2024-04-02T04:43:21.332 INFO:tasks.cephadm:First mon is mon.a on smithi086 2024-04-02T04:43:21.332 INFO:tasks.cephadm:First mgr is y 2024-04-02T04:43:21.333 INFO:tasks.cephadm:Normalizing hostnames... 2024-04-02T04:43:21.333 DEBUG:teuthology.orchestra.run.smithi086:> sudo hostname $(hostname -s) 2024-04-02T04:43:21.361 DEBUG:teuthology.orchestra.run.smithi192:> sudo hostname $(hostname -s) 2024-04-02T04:43:21.389 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-04-02T04:43:21.390 DEBUG:teuthology.orchestra.run.smithi086:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-02T04:43:21.641 INFO:teuthology.orchestra.run.smithi086.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 2 04:43 /home/ubuntu/cephtest/cephadm 2024-04-02T04:43:21.642 DEBUG:teuthology.orchestra.run.smithi192:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-02T04:43:21.881 INFO:teuthology.orchestra.run.smithi192.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 2 04:43 /home/ubuntu/cephtest/cephadm 2024-04-02T04:43:21.882 DEBUG:teuthology.orchestra.run.smithi086:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-02T04:43:21.898 DEBUG:teuthology.orchestra.run.smithi192:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-02T04:43:21.924 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-04-02T04:43:21.925 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-02T04:43:21.940 DEBUG:teuthology.orchestra.run.smithi192:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-02T04:43:22.795 INFO:teuthology.orchestra.run.smithi192.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-02T04:43:22.827 INFO:teuthology.orchestra.run.smithi086.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-02T04:43:40.524 INFO:teuthology.orchestra.run.smithi192.stdout:{ 2024-04-02T04:43:40.525 INFO:teuthology.orchestra.run.smithi192.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-02T04:43:40.525 INFO:teuthology.orchestra.run.smithi192.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-02T04:43:40.525 INFO:teuthology.orchestra.run.smithi192.stdout: "repo_digests": [ 2024-04-02T04:43:40.525 INFO:teuthology.orchestra.run.smithi192.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-02T04:43:40.525 INFO:teuthology.orchestra.run.smithi192.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-02T04:43:40.525 INFO:teuthology.orchestra.run.smithi192.stdout: ] 2024-04-02T04:43:40.525 INFO:teuthology.orchestra.run.smithi192.stdout:} 2024-04-02T04:43:41.799 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T04:43:41.799 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-02T04:43:41.799 INFO:teuthology.orchestra.run.smithi086.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-02T04:43:41.799 INFO:teuthology.orchestra.run.smithi086.stdout: "repo_digests": [ 2024-04-02T04:43:41.799 INFO:teuthology.orchestra.run.smithi086.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-02T04:43:41.800 INFO:teuthology.orchestra.run.smithi086.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-02T04:43:41.800 INFO:teuthology.orchestra.run.smithi086.stdout: ] 2024-04-02T04:43:41.800 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T04:43:41.817 DEBUG:teuthology.orchestra.run.smithi086:> sudo mkdir -p /etc/ceph 2024-04-02T04:43:41.860 DEBUG:teuthology.orchestra.run.smithi192:> sudo mkdir -p /etc/ceph 2024-04-02T04:43:41.897 DEBUG:teuthology.orchestra.run.smithi086:> sudo chmod 777 /etc/ceph 2024-04-02T04:43:41.927 DEBUG:teuthology.orchestra.run.smithi192:> sudo chmod 777 /etc/ceph 2024-04-02T04:43:41.966 INFO:tasks.cephadm:Writing seed config... 2024-04-02T04:43:41.968 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2024-04-02T04:43:41.968 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-04-02T04:43:41.968 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-04-02T04:43:41.968 INFO:tasks.cephadm: override: [mgr] mgr/cephadm/use_agent = False 2024-04-02T04:43:41.968 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-04-02T04:43:41.968 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-04-02T04:43:41.968 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-04-02T04:43:41.968 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-04-02T04:43:41.968 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-04-02T04:43:41.969 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-04-02T04:43:41.969 DEBUG:teuthology.orchestra.run.smithi086:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-04-02T04:43:41.984 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = true bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 mgr/cephadm/use_agent = False [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-04-02T04:43:41.984 DEBUG:teuthology.orchestra.run.smithi086:mon.a> sudo journalctl -f -n 0 -u ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mon.a.service 2024-04-02T04:43:42.027 DEBUG:teuthology.orchestra.run.smithi086:mgr.y> sudo journalctl -f -n 0 -u ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mgr.y.service 2024-04-02T04:43:42.070 INFO:tasks.cephadm:Bootstrapping... 2024-04-02T04:43:42.070 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.86 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-04-02T04:43:42.264 INFO:teuthology.orchestra.run.smithi086.stderr:-------------------------------------------------------------------------------- 2024-04-02T04:43:42.264 INFO:teuthology.orchestra.run.smithi086.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '8862e8fe-f0ab-11ee-b647-cb9ed24678a4', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.86', '--skip-admin-label'] 2024-04-02T04:43:42.286 INFO:teuthology.orchestra.run.smithi086.stderr:/bin/podman: 4.9.4-dev 2024-04-02T04:43:42.289 INFO:teuthology.orchestra.run.smithi086.stderr:Verifying podman|docker is present... 2024-04-02T04:43:42.311 INFO:teuthology.orchestra.run.smithi086.stderr:/bin/podman: 4.9.4-dev 2024-04-02T04:43:42.312 INFO:teuthology.orchestra.run.smithi086.stderr:Verifying lvm2 is present... 2024-04-02T04:43:42.312 INFO:teuthology.orchestra.run.smithi086.stderr:Verifying time synchronization is in place... 2024-04-02T04:43:42.319 INFO:teuthology.orchestra.run.smithi086.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-02T04:43:42.325 INFO:teuthology.orchestra.run.smithi086.stderr:systemctl: inactive 2024-04-02T04:43:42.332 INFO:teuthology.orchestra.run.smithi086.stderr:systemctl: enabled 2024-04-02T04:43:42.337 INFO:teuthology.orchestra.run.smithi086.stderr:systemctl: active 2024-04-02T04:43:42.338 INFO:teuthology.orchestra.run.smithi086.stderr:Unit chronyd.service is enabled and running 2024-04-02T04:43:42.338 INFO:teuthology.orchestra.run.smithi086.stderr:Repeating the final host check... 2024-04-02T04:43:42.359 INFO:teuthology.orchestra.run.smithi086.stderr:/bin/podman: 4.9.4-dev 2024-04-02T04:43:42.361 INFO:teuthology.orchestra.run.smithi086.stderr:podman (/bin/podman) version 4.9.4 is present 2024-04-02T04:43:42.361 INFO:teuthology.orchestra.run.smithi086.stderr:systemctl is present 2024-04-02T04:43:42.361 INFO:teuthology.orchestra.run.smithi086.stderr:lvcreate is present 2024-04-02T04:43:42.367 INFO:teuthology.orchestra.run.smithi086.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-02T04:43:42.373 INFO:teuthology.orchestra.run.smithi086.stderr:systemctl: inactive 2024-04-02T04:43:42.379 INFO:teuthology.orchestra.run.smithi086.stderr:systemctl: enabled 2024-04-02T04:43:42.385 INFO:teuthology.orchestra.run.smithi086.stderr:systemctl: active 2024-04-02T04:43:42.386 INFO:teuthology.orchestra.run.smithi086.stderr:Unit chronyd.service is enabled and running 2024-04-02T04:43:42.386 INFO:teuthology.orchestra.run.smithi086.stderr:Host looks OK 2024-04-02T04:43:42.386 INFO:teuthology.orchestra.run.smithi086.stderr:Cluster fsid: 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 2024-04-02T04:43:42.386 INFO:teuthology.orchestra.run.smithi086.stderr:Acquiring lock 140599741767152 on /run/cephadm/8862e8fe-f0ab-11ee-b647-cb9ed24678a4.lock 2024-04-02T04:43:42.386 INFO:teuthology.orchestra.run.smithi086.stderr:Lock 140599741767152 acquired on /run/cephadm/8862e8fe-f0ab-11ee-b647-cb9ed24678a4.lock 2024-04-02T04:43:42.387 INFO:teuthology.orchestra.run.smithi086.stderr:Verifying IP 172.21.15.86 port 3300 ... 2024-04-02T04:43:42.387 INFO:teuthology.orchestra.run.smithi086.stderr:Verifying IP 172.21.15.86 port 6789 ... 2024-04-02T04:43:42.387 INFO:teuthology.orchestra.run.smithi086.stderr:Base mon IP is 172.21.15.86, final addrv is [v2:172.21.15.86:3300,v1:172.21.15.86:6789] 2024-04-02T04:43:42.389 INFO:teuthology.orchestra.run.smithi086.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.86 metric 100 2024-04-02T04:43:42.389 INFO:teuthology.orchestra.run.smithi086.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.86 metric 100 2024-04-02T04:43:42.392 INFO:teuthology.orchestra.run.smithi086.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-04-02T04:43:42.392 INFO:teuthology.orchestra.run.smithi086.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-04-02T04:43:42.392 INFO:teuthology.orchestra.run.smithi086.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 1024 expires 1787sec hoplimit 64 pref medium 2024-04-02T04:43:42.395 INFO:teuthology.orchestra.run.smithi086.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-04-02T04:43:42.395 INFO:teuthology.orchestra.run.smithi086.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-04-02T04:43:42.395 INFO:teuthology.orchestra.run.smithi086.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-02T04:43:42.395 INFO:teuthology.orchestra.run.smithi086.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-04-02T04:43:42.395 INFO:teuthology.orchestra.run.smithi086.stderr:/sbin/ip: inet6 fe80::ec4:7aff:fe8f:d809/64 scope link 2024-04-02T04:43:42.395 INFO:teuthology.orchestra.run.smithi086.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-02T04:43:42.396 INFO:teuthology.orchestra.run.smithi086.stderr:Mon IP `172.21.15.86` is in CIDR network `172.21.0.0/20` 2024-04-02T04:43:42.396 INFO:teuthology.orchestra.run.smithi086.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-04-02T04:43:42.396 INFO:teuthology.orchestra.run.smithi086.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-02T04:43:42.494 INFO:teuthology.orchestra.run.smithi086.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-04-02T04:43:43.020 INFO:teuthology.orchestra.run.smithi086.stderr:/bin/podman: Getting image source signatures 2024-04-02T04:43:43.021 INFO:teuthology.orchestra.run.smithi086.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-04-02T04:43:43.021 INFO:teuthology.orchestra.run.smithi086.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-04-02T04:43:43.021 INFO:teuthology.orchestra.run.smithi086.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-04-02T04:43:43.021 INFO:teuthology.orchestra.run.smithi086.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-04-02T04:43:43.021 INFO:teuthology.orchestra.run.smithi086.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-04-02T04:43:43.021 INFO:teuthology.orchestra.run.smithi086.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-02T04:43:43.353 INFO:teuthology.orchestra.run.smithi086.stderr:/bin/podman: Writing manifest to image destination 2024-04-02T04:43:43.506 INFO:teuthology.orchestra.run.smithi086.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-02T04:43:43.934 INFO:teuthology.orchestra.run.smithi086.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-02T04:43:44.413 INFO:teuthology.orchestra.run.smithi086.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-02T04:43:44.413 INFO:teuthology.orchestra.run.smithi086.stderr:Extracting ceph user uid/gid from container image... 2024-04-02T04:43:44.866 INFO:teuthology.orchestra.run.smithi086.stderr:stat: 167 167 2024-04-02T04:43:45.277 INFO:teuthology.orchestra.run.smithi086.stderr:Creating initial keys... 2024-04-02T04:43:45.676 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph-authtool: AQABjQtmi1kmIxAAAgKMqWvO5CQXY6X/yuY3tg== 2024-04-02T04:43:46.565 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph-authtool: AQACjQtmkSliHRAAmuV5Ly+nDwRJI/3B/69a+A== 2024-04-02T04:43:47.511 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph-authtool: AQADjQtmP78+GRAAANiiXmHeooaOOZjN3NweSA== 2024-04-02T04:43:47.942 INFO:teuthology.orchestra.run.smithi086.stderr:Creating initial monmap... 2024-04-02T04:43:48.333 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-02T04:43:48.333 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-04-02T04:43:48.334 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 2024-04-02T04:43:48.334 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-02T04:43:48.863 INFO:teuthology.orchestra.run.smithi086.stderr:monmaptool for a [v2:172.21.15.86:3300,v1:172.21.15.86:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-02T04:43:48.863 INFO:teuthology.orchestra.run.smithi086.stderr:setting min_mon_release = octopus 2024-04-02T04:43:48.863 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/monmaptool: set fsid to 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 2024-04-02T04:43:48.864 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-02T04:43:48.864 INFO:teuthology.orchestra.run.smithi086.stderr: 2024-04-02T04:43:48.864 INFO:teuthology.orchestra.run.smithi086.stderr:Creating mon... 2024-04-02T04:43:49.826 INFO:teuthology.orchestra.run.smithi086.stderr:create mon.a on 2024-04-02T04:43:50.042 INFO:teuthology.orchestra.run.smithi086.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-04-02T04:43:50.202 INFO:teuthology.orchestra.run.smithi086.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4.target → /etc/systemd/system/ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4.target. 2024-04-02T04:43:50.202 INFO:teuthology.orchestra.run.smithi086.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4.target → /etc/systemd/system/ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4.target. 2024-04-02T04:43:50.496 INFO:teuthology.orchestra.run.smithi086.stderr:systemctl: Failed to reset failed state of unit ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mon.a.service: Unit ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mon.a.service not loaded. 2024-04-02T04:43:50.503 INFO:teuthology.orchestra.run.smithi086.stderr:systemctl: Created symlink /etc/systemd/system/ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4.target.wants/ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mon.a.service → /etc/systemd/system/ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@.service. 2024-04-02T04:43:51.166 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:51 smithi086 podman[24450]: 2024-04-02 04:43:51.120411047 +0000 UTC m=+0.345648556 container start 346cb27a85311d80916a5d17f140b5d939fe1f3c546442b835edb9e6afab3277 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-a, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., io.openshift.expose-services=, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, io.openshift.tags=base centos centos-stream, vcs-type=git, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-02T04:43:51.232 INFO:teuthology.orchestra.run.smithi086.stderr:systemctl: disabled 2024-04-02T04:43:51.238 INFO:teuthology.orchestra.run.smithi086.stderr:systemctl: inactive 2024-04-02T04:43:51.239 INFO:teuthology.orchestra.run.smithi086.stderr:firewalld.service is not enabled 2024-04-02T04:43:51.239 INFO:teuthology.orchestra.run.smithi086.stderr:Not possible to enable service . firewalld.service is not available 2024-04-02T04:43:51.239 INFO:teuthology.orchestra.run.smithi086.stderr:Waiting for mon to start... 2024-04-02T04:43:51.240 INFO:teuthology.orchestra.run.smithi086.stderr:Waiting for mon... 2024-04-02T04:43:51.446 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:51 smithi086 ceph-mon[24468]: mkfs 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 2024-04-02T04:43:51.446 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:51 smithi086 ceph-mon[24468]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-02T04:43:51.446 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:51 smithi086 bash[24450]: 346cb27a85311d80916a5d17f140b5d939fe1f3c546442b835edb9e6afab3277 2024-04-02T04:43:51.447 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:51 smithi086 systemd[1]: Started Ceph mon.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T04:43:51.719 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: cluster: 2024-04-02T04:43:51.719 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: id: 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 2024-04-02T04:43:51.719 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-04-02T04:43:51.719 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: 2024-04-02T04:43:51.719 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: services: 2024-04-02T04:43:51.719 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.556326s) 2024-04-02T04:43:51.719 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: mgr: no daemons active 2024-04-02T04:43:51.719 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-04-02T04:43:51.719 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: 2024-04-02T04:43:51.719 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: data: 2024-04-02T04:43:51.719 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-04-02T04:43:51.719 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-04-02T04:43:51.720 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-04-02T04:43:51.720 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: pgs: 2024-04-02T04:43:51.720 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: 2024-04-02T04:43:52.227 INFO:teuthology.orchestra.run.smithi086.stderr:mon is available 2024-04-02T04:43:52.228 INFO:teuthology.orchestra.run.smithi086.stderr:Assimilating anything we can from ceph.conf... 2024-04-02T04:43:52.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:52 smithi086 ceph-mon[24468]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-02T04:43:52.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:52 smithi086 ceph-mon[24468]: monmap e1: 1 mons at {a=[v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0]} 2024-04-02T04:43:52.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:52 smithi086 ceph-mon[24468]: fsmap 2024-04-02T04:43:52.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:52 smithi086 ceph-mon[24468]: osdmap e1: 0 total, 0 up, 0 in 2024-04-02T04:43:52.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:52 smithi086 ceph-mon[24468]: mgrmap e1: no daemons active 2024-04-02T04:43:52.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:52 smithi086 ceph-mon[24468]: from='client.? 172.21.15.86:0/3437553354' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-02T04:43:52.689 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: 2024-04-02T04:43:52.689 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: [global] 2024-04-02T04:43:52.689 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: fsid = 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 2024-04-02T04:43:52.689 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.86:3300,v1:172.21.15.86:6789] 2024-04-02T04:43:52.689 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-02T04:43:52.690 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-02T04:43:52.690 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-02T04:43:52.690 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-02T04:43:52.690 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: 2024-04-02T04:43:52.690 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: [mgr] 2024-04-02T04:43:52.690 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: mgr/cephadm/use_agent = False 2024-04-02T04:43:52.690 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-02T04:43:52.690 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: 2024-04-02T04:43:52.690 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: [osd] 2024-04-02T04:43:52.690 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-02T04:43:52.690 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-02T04:43:53.267 INFO:teuthology.orchestra.run.smithi086.stderr:Generating new minimal ceph.conf... 2024-04-02T04:43:53.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:53 smithi086 ceph-mon[24468]: from='client.? 172.21.15.86:0/3334165407' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-02T04:43:53.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:53 smithi086 ceph-mon[24468]: from='client.? 172.21.15.86:0/3334165407' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-02T04:43:54.289 INFO:teuthology.orchestra.run.smithi086.stderr:Restarting the monitor... 2024-04-02T04:43:54.311 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:54 smithi086 ceph-mon[24468]: from='client.? 172.21.15.86:0/4286765569' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:43:54.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:54 smithi086 systemd[1]: Stopping Ceph mon.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T04:43:54.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:54 smithi086 bash[24686]: Error: no container with name or ID "ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon.a" found: no such container 2024-04-02T04:43:54.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:54 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-a[24464]: 2024-04-02T04:43:54.428+0000 7f704a073700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T04:43:54.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:54 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-a[24464]: 2024-04-02T04:43:54.428+0000 7f704a073700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-04-02T04:43:54.930 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:54 smithi086 podman[24697]: 2024-04-02 04:43:54.592346124 +0000 UTC m=+0.248600171 container died 346cb27a85311d80916a5d17f140b5d939fe1f3c546442b835edb9e6afab3277 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-a, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, GIT_CLEAN=True, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, vcs-type=git, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, name=centos-stream, io.buildah.version=1.19.8, release=754, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8) 2024-04-02T04:43:54.931 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:54 smithi086 podman[24697]: 2024-04-02 04:43:54.928160727 +0000 UTC m=+0.584414786 container cleanup 346cb27a85311d80916a5d17f140b5d939fe1f3c546442b835edb9e6afab3277 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-a, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, vcs-type=git, version=8, GIT_CLEAN=True, name=centos-stream, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=) 2024-04-02T04:43:55.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:54 smithi086 bash[24697]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-a 2024-04-02T04:43:55.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 podman[24711]: 2024-04-02 04:43:55.095644423 +0000 UTC m=+0.500677765 container remove 346cb27a85311d80916a5d17f140b5d939fe1f3c546442b835edb9e6afab3277 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-a, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, version=8, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, name=centos-stream, io.buildah.version=1.19.8, vcs-type=git, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-04-02T04:43:55.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 bash[24723]: Error: no container with name or ID "ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon.a" found: no such container 2024-04-02T04:43:55.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mon.a.service: Deactivated successfully. 2024-04-02T04:43:55.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 systemd[1]: Stopped Ceph mon.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T04:43:55.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 systemd[1]: Starting Ceph mon.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T04:43:55.612 INFO:teuthology.orchestra.run.smithi086.stderr:Setting mon public_network to 172.21.0.0/20 2024-04-02T04:43:55.633 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 podman[24792]: 2024-04-02 04:43:55.362797875 +0000 UTC m=+0.111222097 container create b6c507ab96f87cb05b52f8388f6b58a90d5c1eee4c4cf01f4b541161b2096dee (image=quay.io/ceph/ceph:v17.2.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-a, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, ceph=True, GIT_BRANCH=HEAD, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, distribution-scope=public, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, release=754, io.k8s.display-name=CentOS Stream 8, name=centos-stream, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream) 2024-04-02T04:43:55.633 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 podman[24792]: 2024-04-02 04:43:55.26934803 +0000 UTC m=+0.017772257 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-04-02T04:43:55.633 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 podman[24792]: 2024-04-02 04:43:55.540219095 +0000 UTC m=+0.288643321 container init b6c507ab96f87cb05b52f8388f6b58a90d5c1eee4c4cf01f4b541161b2096dee (image=quay.io/ceph/ceph:v17.2.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-a, release=754, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , name=centos-stream, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, ceph=True, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-04-02T04:43:55.633 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 podman[24792]: 2024-04-02 04:43:55.543161644 +0000 UTC m=+0.291585865 container start b6c507ab96f87cb05b52f8388f6b58a90d5c1eee4c4cf01f4b541161b2096dee (image=quay.io/ceph/ceph:v17.2.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-a, RELEASE=HEAD, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , io.openshift.expose-services=, io.buildah.version=1.19.8, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, vcs-type=git, name=centos-stream, architecture=x86_64, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, build-date=2022-05-03T08:36:31.336870, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-02T04:43:55.634 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: set uid:gid to 167:167 (ceph:ceph) 2024-04-02T04:43:55.634 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-04-02T04:43:55.634 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: pidfile_write: ignore empty --pid-file 2024-04-02T04:43:55.634 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: load: jerasure load: lrc 2024-04-02T04:43:55.634 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: RocksDB version: 6.15.5 2024-04-02T04:43:55.634 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-02T04:43:55.634 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Compile date Apr 18 2022 2024-04-02T04:43:55.634 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: DB SUMMARY 2024-04-02T04:43:55.634 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: DB Session ID: MFNGLRD9O9OB0KFWH9JQ 2024-04-02T04:43:55.634 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: CURRENT file: CURRENT 2024-04-02T04:43:55.634 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: IDENTITY file: IDENTITY 2024-04-02T04:43:55.634 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-04-02T04:43:55.634 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-04-02T04:43:55.635 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 82637 ; 2024-04-02T04:43:55.635 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.error_if_exists: 0 2024-04-02T04:43:55.635 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.create_if_missing: 0 2024-04-02T04:43:55.635 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.paranoid_checks: 1 2024-04-02T04:43:55.635 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-02T04:43:55.635 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.env: 0x55a83dec6860 2024-04-02T04:43:55.635 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.fs: Posix File System 2024-04-02T04:43:55.635 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.info_log: 0x55a83e9addc0 2024-04-02T04:43:55.635 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.max_file_opening_threads: 16 2024-04-02T04:43:55.635 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.statistics: (nil) 2024-04-02T04:43:55.635 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.use_fsync: 0 2024-04-02T04:43:55.635 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.max_log_file_size: 0 2024-04-02T04:43:55.635 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-02T04:43:55.636 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-02T04:43:55.636 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.keep_log_file_num: 1000 2024-04-02T04:43:55.636 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.recycle_log_file_num: 0 2024-04-02T04:43:55.637 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.allow_fallocate: 1 2024-04-02T04:43:55.637 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.allow_mmap_reads: 0 2024-04-02T04:43:55.637 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.allow_mmap_writes: 0 2024-04-02T04:43:55.637 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.use_direct_reads: 0 2024-04-02T04:43:55.637 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-02T04:43:55.637 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.create_missing_column_families: 0 2024-04-02T04:43:55.637 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.db_log_dir: 2024-04-02T04:43:55.637 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-04-02T04:43:55.637 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-02T04:43:55.637 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-02T04:43:55.637 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-02T04:43:55.637 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-02T04:43:55.637 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-02T04:43:55.638 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-02T04:43:55.638 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.advise_random_on_open: 1 2024-04-02T04:43:55.638 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.db_write_buffer_size: 0 2024-04-02T04:43:55.638 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.write_buffer_manager: 0x55a83ea9e2a0 2024-04-02T04:43:55.638 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-02T04:43:55.638 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-02T04:43:55.638 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-02T04:43:55.638 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-02T04:43:55.638 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.rate_limiter: (nil) 2024-04-02T04:43:55.638 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-02T04:43:55.638 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.wal_recovery_mode: 2 2024-04-02T04:43:55.638 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.enable_thread_tracking: 0 2024-04-02T04:43:55.638 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.enable_pipelined_write: 0 2024-04-02T04:43:55.639 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.unordered_write: 0 2024-04-02T04:43:55.639 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-02T04:43:55.639 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-02T04:43:55.639 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-02T04:43:55.639 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-02T04:43:55.639 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.row_cache: None 2024-04-02T04:43:55.639 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.wal_filter: None 2024-04-02T04:43:55.639 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-02T04:43:55.639 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.allow_ingest_behind: 0 2024-04-02T04:43:55.639 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.preserve_deletes: 0 2024-04-02T04:43:55.639 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.two_write_queues: 0 2024-04-02T04:43:55.639 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.manual_wal_flush: 0 2024-04-02T04:43:55.639 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.atomic_flush: 0 2024-04-02T04:43:55.639 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-02T04:43:55.639 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-02T04:43:55.640 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-02T04:43:55.640 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.log_readahead_size: 0 2024-04-02T04:43:55.640 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-02T04:43:55.640 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.best_efforts_recovery: 0 2024-04-02T04:43:55.640 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-02T04:43:55.640 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-02T04:43:55.640 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.allow_data_in_errors: 0 2024-04-02T04:43:55.640 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.db_host_id: __hostname__ 2024-04-02T04:43:55.640 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.max_background_jobs: 2 2024-04-02T04:43:55.640 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.max_background_compactions: -1 2024-04-02T04:43:55.640 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.max_subcompactions: 1 2024-04-02T04:43:55.640 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-02T04:43:55.640 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-02T04:43:55.640 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-02T04:43:55.641 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.max_total_wal_size: 0 2024-04-02T04:43:55.641 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-02T04:43:55.641 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-02T04:43:55.641 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-02T04:43:55.641 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-02T04:43:55.641 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.max_open_files: -1 2024-04-02T04:43:55.641 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.bytes_per_sync: 0 2024-04-02T04:43:55.641 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-02T04:43:55.641 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-02T04:43:55.641 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.compaction_readahead_size: 0 2024-04-02T04:43:55.641 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.max_background_flushes: -1 2024-04-02T04:43:55.641 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Compression algorithms supported: 2024-04-02T04:43:55.641 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-02T04:43:55.641 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: kZSTD supported: 0 2024-04-02T04:43:55.641 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: kXpressCompression supported: 0 2024-04-02T04:43:55.642 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: kLZ4HCCompression supported: 1 2024-04-02T04:43:55.642 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: kLZ4Compression supported: 1 2024-04-02T04:43:55.642 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: kBZip2Compression supported: 0 2024-04-02T04:43:55.642 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: kZlibCompression supported: 1 2024-04-02T04:43:55.642 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: kSnappyCompression supported: 1 2024-04-02T04:43:55.642 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-02T04:43:55.642 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-04-02T04:43:55.642 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-04-02T04:43:55.642 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-02T04:43:55.642 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.merge_operator: 2024-04-02T04:43:55.642 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.compaction_filter: None 2024-04-02T04:43:55.642 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.compaction_filter_factory: None 2024-04-02T04:43:55.642 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.sst_partitioner_factory: None 2024-04-02T04:43:55.642 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-02T04:43:55.643 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-02T04:43:55.643 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55a83e979d00) 2024-04-02T04:43:55.643 INFO:journalctl@ceph.mon.a.smithi086.stdout: cache_index_and_filter_blocks: 1 2024-04-02T04:43:55.643 INFO:journalctl@ceph.mon.a.smithi086.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-02T04:43:55.643 INFO:journalctl@ceph.mon.a.smithi086.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-02T04:43:55.643 INFO:journalctl@ceph.mon.a.smithi086.stdout: pin_top_level_index_and_filter: 1 2024-04-02T04:43:55.643 INFO:journalctl@ceph.mon.a.smithi086.stdout: index_type: 0 2024-04-02T04:43:55.643 INFO:journalctl@ceph.mon.a.smithi086.stdout: data_block_index_type: 0 2024-04-02T04:43:55.643 INFO:journalctl@ceph.mon.a.smithi086.stdout: index_shortening: 1 2024-04-02T04:43:55.643 INFO:journalctl@ceph.mon.a.smithi086.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-02T04:43:55.643 INFO:journalctl@ceph.mon.a.smithi086.stdout: hash_index_allow_collision: 1 2024-04-02T04:43:55.643 INFO:journalctl@ceph.mon.a.smithi086.stdout: checksum: 1 2024-04-02T04:43:55.643 INFO:journalctl@ceph.mon.a.smithi086.stdout: no_block_cache: 0 2024-04-02T04:43:55.643 INFO:journalctl@ceph.mon.a.smithi086.stdout: block_cache: 0x55a83e9e4170 2024-04-02T04:43:55.643 INFO:journalctl@ceph.mon.a.smithi086.stdout: block_cache_name: BinnedLRUCache 2024-04-02T04:43:55.644 INFO:journalctl@ceph.mon.a.smithi086.stdout: block_cache_options: 2024-04-02T04:43:55.644 INFO:journalctl@ceph.mon.a.smithi086.stdout: capacity : 536870912 2024-04-02T04:43:55.644 INFO:journalctl@ceph.mon.a.smithi086.stdout: num_shard_bits : 4 2024-04-02T04:43:55.644 INFO:journalctl@ceph.mon.a.smithi086.stdout: strict_capacity_limit : 0 2024-04-02T04:43:55.644 INFO:journalctl@ceph.mon.a.smithi086.stdout: high_pri_pool_ratio: 0.000 2024-04-02T04:43:55.644 INFO:journalctl@ceph.mon.a.smithi086.stdout: block_cache_compressed: (nil) 2024-04-02T04:43:55.644 INFO:journalctl@ceph.mon.a.smithi086.stdout: persistent_cache: (nil) 2024-04-02T04:43:55.644 INFO:journalctl@ceph.mon.a.smithi086.stdout: block_size: 4096 2024-04-02T04:43:55.644 INFO:journalctl@ceph.mon.a.smithi086.stdout: block_size_deviation: 10 2024-04-02T04:43:55.644 INFO:journalctl@ceph.mon.a.smithi086.stdout: block_restart_interval: 16 2024-04-02T04:43:55.644 INFO:journalctl@ceph.mon.a.smithi086.stdout: index_block_restart_interval: 1 2024-04-02T04:43:55.644 INFO:journalctl@ceph.mon.a.smithi086.stdout: metadata_block_size: 4096 2024-04-02T04:43:55.644 INFO:journalctl@ceph.mon.a.smithi086.stdout: partition_filters: 0 2024-04-02T04:43:55.644 INFO:journalctl@ceph.mon.a.smithi086.stdout: use_delta_encoding: 1 2024-04-02T04:43:55.644 INFO:journalctl@ceph.mon.a.smithi086.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-02T04:43:55.645 INFO:journalctl@ceph.mon.a.smithi086.stdout: whole_key_filtering: 1 2024-04-02T04:43:55.645 INFO:journalctl@ceph.mon.a.smithi086.stdout: verify_compression: 0 2024-04-02T04:43:55.645 INFO:journalctl@ceph.mon.a.smithi086.stdout: read_amp_bytes_per_bit: 0 2024-04-02T04:43:55.645 INFO:journalctl@ceph.mon.a.smithi086.stdout: format_version: 4 2024-04-02T04:43:55.645 INFO:journalctl@ceph.mon.a.smithi086.stdout: enable_index_compression: 1 2024-04-02T04:43:55.645 INFO:journalctl@ceph.mon.a.smithi086.stdout: block_align: 0 2024-04-02T04:43:55.645 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.write_buffer_size: 33554432 2024-04-02T04:43:55.645 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.max_write_buffer_number: 2 2024-04-02T04:43:55.645 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.compression: NoCompression 2024-04-02T04:43:55.645 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.bottommost_compression: Disabled 2024-04-02T04:43:55.645 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.prefix_extractor: nullptr 2024-04-02T04:43:55.645 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-02T04:43:55.645 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.num_levels: 7 2024-04-02T04:43:55.645 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-02T04:43:55.645 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-02T04:43:55.646 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-02T04:43:55.646 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-02T04:43:55.646 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-02T04:43:55.646 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-02T04:43:55.646 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-02T04:43:55.646 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-02T04:43:55.646 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-02T04:43:55.646 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-02T04:43:55.646 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-02T04:43:55.646 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.compression_opts.level: 32767 2024-04-02T04:43:55.646 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.compression_opts.strategy: 0 2024-04-02T04:43:55.646 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-02T04:43:55.646 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-02T04:43:55.646 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-02T04:43:55.647 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.compression_opts.enabled: false 2024-04-02T04:43:55.647 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-02T04:43:55.647 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-02T04:43:55.647 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-02T04:43:55.647 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.target_file_size_base: 67108864 2024-04-02T04:43:55.647 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-02T04:43:55.647 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-02T04:43:55.647 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-02T04:43:55.647 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-02T04:43:55.647 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-02T04:43:55.647 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-02T04:43:55.647 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-02T04:43:55.647 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-02T04:43:55.647 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-02T04:43:55.648 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-02T04:43:55.648 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-02T04:43:55.648 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-02T04:43:55.648 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-02T04:43:55.648 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.arena_block_size: 4194304 2024-04-02T04:43:55.648 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-02T04:43:55.648 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-02T04:43:55.648 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-02T04:43:55.649 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.disable_auto_compactions: 0 2024-04-02T04:43:55.649 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-02T04:43:55.649 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-02T04:43:55.649 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-02T04:43:55.649 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-02T04:43:55.649 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-02T04:43:55.649 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-02T04:43:55.649 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-02T04:43:55.650 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-02T04:43:55.650 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-02T04:43:55.650 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-02T04:43:55.650 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.table_properties_collectors: 2024-04-02T04:43:55.650 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.inplace_update_support: 0 2024-04-02T04:43:55.650 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-02T04:43:55.650 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-02T04:43:55.650 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-02T04:43:55.650 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-02T04:43:55.650 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.bloom_locality: 0 2024-04-02T04:43:55.650 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.max_successive_merges: 0 2024-04-02T04:43:55.650 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-02T04:43:55.650 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.paranoid_file_checks: 0 2024-04-02T04:43:55.650 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.force_consistency_checks: 1 2024-04-02T04:43:55.651 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.report_bg_io_stats: 0 2024-04-02T04:43:55.651 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.ttl: 2592000 2024-04-02T04:43:55.651 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-02T04:43:55.651 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.enable_blob_files: false 2024-04-02T04:43:55.651 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.min_blob_size: 0 2024-04-02T04:43:55.651 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.blob_file_size: 268435456 2024-04-02T04:43:55.651 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-02T04:43:55.651 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-02T04:43:55.651 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-02T04:43:55.651 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-02T04:43:55.651 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-04-02T04:43:55.651 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-04-02T04:43:55.651 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712033035568253, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-04-02T04:43:55.651 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-04-02T04:43:55.651 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-02T04:43:55.651 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712033035568939, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79187, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77450, "index_size": 225, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10027, "raw_average_key_size": 47, "raw_value_size": 71759, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 211, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1712033035, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "209ac2fa-d72e-482f-8f50-19aca96a133c", "db_session_id": "MFNGLRD9O9OB0KFWH9JQ"}} 2024-04-02T04:43:55.652 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-04-02T04:43:55.652 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712033035569298, "job": 1, "event": "recovery_finished"} 2024-04-02T04:43:55.652 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-02T04:43:55.652 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55a83e9caa80 2024-04-02T04:43:55.652 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: DB pointer 0x55a83e9da000 2024-04-02T04:43:55.652 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-04-02T04:43:55.652 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-02T04:43:55.652 INFO:journalctl@ceph.mon.a.smithi086.stdout: ** DB Stats ** 2024-04-02T04:43:55.652 INFO:journalctl@ceph.mon.a.smithi086.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-02T04:43:55.652 INFO:journalctl@ceph.mon.a.smithi086.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-02T04:43:55.652 INFO:journalctl@ceph.mon.a.smithi086.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-02T04:43:55.652 INFO:journalctl@ceph.mon.a.smithi086.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-02T04:43:55.652 INFO:journalctl@ceph.mon.a.smithi086.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-02T04:43:55.652 INFO:journalctl@ceph.mon.a.smithi086.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-02T04:43:55.653 INFO:journalctl@ceph.mon.a.smithi086.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-02T04:43:55.653 INFO:journalctl@ceph.mon.a.smithi086.stdout: 2024-04-02T04:43:55.653 INFO:journalctl@ceph.mon.a.smithi086.stdout: ** Compaction Stats [default] ** 2024-04-02T04:43:55.653 INFO:journalctl@ceph.mon.a.smithi086.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-02T04:43:55.653 INFO:journalctl@ceph.mon.a.smithi086.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T04:43:55.653 INFO:journalctl@ceph.mon.a.smithi086.stdout: L0 2/0 79.09 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 161.0 0.00 0.00 1 0.000 0 0 2024-04-02T04:43:55.653 INFO:journalctl@ceph.mon.a.smithi086.stdout: Sum 2/0 79.09 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 161.0 0.00 0.00 1 0.000 0 0 2024-04-02T04:43:55.653 INFO:journalctl@ceph.mon.a.smithi086.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 161.0 0.00 0.00 1 0.000 0 0 2024-04-02T04:43:55.653 INFO:journalctl@ceph.mon.a.smithi086.stdout: 2024-04-02T04:43:55.653 INFO:journalctl@ceph.mon.a.smithi086.stdout: ** Compaction Stats [default] ** 2024-04-02T04:43:55.653 INFO:journalctl@ceph.mon.a.smithi086.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-02T04:43:55.653 INFO:journalctl@ceph.mon.a.smithi086.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T04:43:55.653 INFO:journalctl@ceph.mon.a.smithi086.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 161.0 0.00 0.00 1 0.000 0 0 2024-04-02T04:43:55.654 INFO:journalctl@ceph.mon.a.smithi086.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-02T04:43:55.654 INFO:journalctl@ceph.mon.a.smithi086.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-02T04:43:55.654 INFO:journalctl@ceph.mon.a.smithi086.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-02T04:43:55.654 INFO:journalctl@ceph.mon.a.smithi086.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-02T04:43:55.654 INFO:journalctl@ceph.mon.a.smithi086.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-02T04:43:55.654 INFO:journalctl@ceph.mon.a.smithi086.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-02T04:43:55.654 INFO:journalctl@ceph.mon.a.smithi086.stdout: Cumulative compaction: 0.00 GB write, 25.67 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T04:43:55.654 INFO:journalctl@ceph.mon.a.smithi086.stdout: Interval compaction: 0.00 GB write, 25.67 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T04:43:55.654 INFO:journalctl@ceph.mon.a.smithi086.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-02T04:43:55.654 INFO:journalctl@ceph.mon.a.smithi086.stdout: 2024-04-02T04:43:55.654 INFO:journalctl@ceph.mon.a.smithi086.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-02T04:43:55.654 INFO:journalctl@ceph.mon.a.smithi086.stdout: 2024-04-02T04:43:55.654 INFO:journalctl@ceph.mon.a.smithi086.stdout: ** Compaction Stats [default] ** 2024-04-02T04:43:55.654 INFO:journalctl@ceph.mon.a.smithi086.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-02T04:43:55.655 INFO:journalctl@ceph.mon.a.smithi086.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T04:43:55.655 INFO:journalctl@ceph.mon.a.smithi086.stdout: L0 2/0 79.09 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 161.0 0.00 0.00 1 0.000 0 0 2024-04-02T04:43:55.655 INFO:journalctl@ceph.mon.a.smithi086.stdout: Sum 2/0 79.09 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 161.0 0.00 0.00 1 0.000 0 0 2024-04-02T04:43:55.655 INFO:journalctl@ceph.mon.a.smithi086.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-02T04:43:55.655 INFO:journalctl@ceph.mon.a.smithi086.stdout: 2024-04-02T04:43:55.655 INFO:journalctl@ceph.mon.a.smithi086.stdout: ** Compaction Stats [default] ** 2024-04-02T04:43:55.655 INFO:journalctl@ceph.mon.a.smithi086.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-02T04:43:55.655 INFO:journalctl@ceph.mon.a.smithi086.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T04:43:55.655 INFO:journalctl@ceph.mon.a.smithi086.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 161.0 0.00 0.00 1 0.000 0 0 2024-04-02T04:43:55.655 INFO:journalctl@ceph.mon.a.smithi086.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-02T04:43:55.655 INFO:journalctl@ceph.mon.a.smithi086.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-02T04:43:55.655 INFO:journalctl@ceph.mon.a.smithi086.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-02T04:43:55.655 INFO:journalctl@ceph.mon.a.smithi086.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-02T04:43:55.655 INFO:journalctl@ceph.mon.a.smithi086.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-02T04:43:55.656 INFO:journalctl@ceph.mon.a.smithi086.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-02T04:43:55.656 INFO:journalctl@ceph.mon.a.smithi086.stdout: Cumulative compaction: 0.00 GB write, 25.38 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T04:43:55.656 INFO:journalctl@ceph.mon.a.smithi086.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T04:43:55.656 INFO:journalctl@ceph.mon.a.smithi086.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-02T04:43:55.656 INFO:journalctl@ceph.mon.a.smithi086.stdout: 2024-04-02T04:43:55.656 INFO:journalctl@ceph.mon.a.smithi086.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-02T04:43:55.656 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: starting mon.a rank 0 at public addrs [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] at bind addrs [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 2024-04-02T04:43:55.656 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: mon.a@-1(???) e1 preinit fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 2024-04-02T04:43:55.656 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: mon.a@-1(???).mds e1 new map 2024-04-02T04:43:55.656 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: mon.a@-1(???).mds e1 print_map 2024-04-02T04:43:55.656 INFO:journalctl@ceph.mon.a.smithi086.stdout: e1 2024-04-02T04:43:55.656 INFO:journalctl@ceph.mon.a.smithi086.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-02T04:43:55.656 INFO:journalctl@ceph.mon.a.smithi086.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-02T04:43:55.656 INFO:journalctl@ceph.mon.a.smithi086.stdout: legacy client fscid: -1 2024-04-02T04:43:55.657 INFO:journalctl@ceph.mon.a.smithi086.stdout: 2024-04-02T04:43:55.657 INFO:journalctl@ceph.mon.a.smithi086.stdout: No filesystems configured 2024-04-02T04:43:55.657 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-02T04:43:55.657 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-02T04:43:55.657 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-02T04:43:55.657 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-02T04:43:55.657 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-04-02T04:43:55.657 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: expand_channel_meta expand map: {default=false} 2024-04-02T04:43:55.657 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: expand_channel_meta from 'false' to 'false' 2024-04-02T04:43:55.657 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: expand_channel_meta expanded map: {default=false} 2024-04-02T04:43:55.657 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: expand_channel_meta expand map: {default=info} 2024-04-02T04:43:55.657 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: expand_channel_meta from 'info' to 'info' 2024-04-02T04:43:55.657 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: expand_channel_meta expanded map: {default=info} 2024-04-02T04:43:55.657 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: expand_channel_meta expand map: {default=daemon} 2024-04-02T04:43:55.658 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: expand_channel_meta from 'daemon' to 'daemon' 2024-04-02T04:43:55.658 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: expand_channel_meta expanded map: {default=daemon} 2024-04-02T04:43:55.658 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: expand_channel_meta expand map: {default=debug} 2024-04-02T04:43:55.658 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: expand_channel_meta from 'debug' to 'debug' 2024-04-02T04:43:55.658 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: expand_channel_meta expanded map: {default=debug} 2024-04-02T04:43:55.658 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 bash[24792]: b6c507ab96f87cb05b52f8388f6b58a90d5c1eee4c4cf01f4b541161b2096dee 2024-04-02T04:43:55.658 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 systemd[1]: Started Ceph mon.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T04:43:55.658 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-02T04:43:55.658 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: monmap e1: 1 mons at {a=[v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0]} 2024-04-02T04:43:55.658 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: fsmap 2024-04-02T04:43:55.658 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: osdmap e1: 0 total, 0 up, 0 in 2024-04-02T04:43:55.658 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:55 smithi086 ceph-mon[24811]: mgrmap e1: no daemons active 2024-04-02T04:43:56.601 INFO:teuthology.orchestra.run.smithi086.stderr:Wrote config to /etc/ceph/ceph.conf 2024-04-02T04:43:56.601 INFO:teuthology.orchestra.run.smithi086.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-04-02T04:43:56.601 INFO:teuthology.orchestra.run.smithi086.stderr:Creating mgr... 2024-04-02T04:43:56.601 INFO:teuthology.orchestra.run.smithi086.stderr:Verifying port 9283 ... 2024-04-02T04:43:56.770 INFO:teuthology.orchestra.run.smithi086.stderr:systemctl: Failed to reset failed state of unit ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mgr.y.service: Unit ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mgr.y.service not loaded. 2024-04-02T04:43:56.777 INFO:teuthology.orchestra.run.smithi086.stderr:systemctl: Created symlink /etc/systemd/system/ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4.target.wants/ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mgr.y.service → /etc/systemd/system/ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@.service. 2024-04-02T04:43:57.068 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:43:56 smithi086 systemd[1]: Starting Ceph mgr.y for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T04:43:57.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:57 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/494209034' entity='client.admin' 2024-04-02T04:43:57.333 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:43:57 smithi086 podman[25025]: 2024-04-02 04:43:57.045635658 +0000 UTC m=+0.018320676 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-04-02T04:43:57.333 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:43:57 smithi086 podman[25025]: 2024-04-02 04:43:57.171751113 +0000 UTC m=+0.144436132 container create 02f336a19fd69627a57257bbda89253ad8dfb6c357f5d7890e0b5b4083eeb81b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., ceph=True, name=centos-stream, release=754, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, GIT_CLEAN=True, vcs-type=git, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=) 2024-04-02T04:43:57.505 INFO:teuthology.orchestra.run.smithi086.stderr:systemctl: disabled 2024-04-02T04:43:57.512 INFO:teuthology.orchestra.run.smithi086.stderr:systemctl: inactive 2024-04-02T04:43:57.513 INFO:teuthology.orchestra.run.smithi086.stderr:firewalld.service is not enabled 2024-04-02T04:43:57.513 INFO:teuthology.orchestra.run.smithi086.stderr:Not possible to enable service . firewalld.service is not available 2024-04-02T04:43:57.521 INFO:teuthology.orchestra.run.smithi086.stderr:systemctl: disabled 2024-04-02T04:43:57.528 INFO:teuthology.orchestra.run.smithi086.stderr:systemctl: inactive 2024-04-02T04:43:57.529 INFO:teuthology.orchestra.run.smithi086.stderr:firewalld.service is not enabled 2024-04-02T04:43:57.529 INFO:teuthology.orchestra.run.smithi086.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-04-02T04:43:57.529 INFO:teuthology.orchestra.run.smithi086.stderr:Waiting for mgr to start... 2024-04-02T04:43:57.529 INFO:teuthology.orchestra.run.smithi086.stderr:Waiting for mgr... 2024-04-02T04:43:57.644 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:43:57 smithi086 podman[25025]: 2024-04-02 04:43:57.365584227 +0000 UTC m=+0.338269258 container init 02f336a19fd69627a57257bbda89253ad8dfb6c357f5d7890e0b5b4083eeb81b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, version=8, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, release=754, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, vcs-type=git, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True) 2024-04-02T04:43:57.644 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:43:57 smithi086 podman[25025]: 2024-04-02 04:43:57.368109536 +0000 UTC m=+0.340794561 container start 02f336a19fd69627a57257bbda89253ad8dfb6c357f5d7890e0b5b4083eeb81b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, distribution-scope=public, io.openshift.tags=base centos centos-stream, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, version=8, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0) 2024-04-02T04:43:57.644 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:43:57 smithi086 bash[25025]: 02f336a19fd69627a57257bbda89253ad8dfb6c357f5d7890e0b5b4083eeb81b 2024-04-02T04:43:57.644 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:43:57 smithi086 systemd[1]: Started Ceph mgr.y for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T04:43:57.953 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:43:57 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:43:57.857+0000 7f269ff37000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-02T04:43:57.953 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:43:57 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:43:57.949+0000 7f269ff37000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-02T04:43:58.060 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: 2024-04-02T04:43:58.060 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: { 2024-04-02T04:43:58.060 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "fsid": "8862e8fe-f0ab-11ee-b647-cb9ed24678a4", 2024-04-02T04:43:58.060 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "health": { 2024-04-02T04:43:58.060 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-02T04:43:58.060 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "checks": {}, 2024-04-02T04:43:58.060 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "mutes": [] 2024-04-02T04:43:58.060 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: }, 2024-04-02T04:43:58.060 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-02T04:43:58.061 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "quorum": [ 2024-04-02T04:43:58.061 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: 0 2024-04-02T04:43:58.061 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: ], 2024-04-02T04:43:58.061 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-02T04:43:58.061 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "a" 2024-04-02T04:43:58.061 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: ], 2024-04-02T04:43:58.061 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-04-02T04:43:58.061 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "monmap": { 2024-04-02T04:43:58.061 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T04:43:58.061 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-02T04:43:58.061 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-02T04:43:58.061 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: }, 2024-04-02T04:43:58.061 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "osdmap": { 2024-04-02T04:43:58.061 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T04:43:58.061 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-02T04:43:58.062 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-02T04:43:58.062 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-02T04:43:58.062 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-02T04:43:58.062 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-02T04:43:58.062 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-02T04:43:58.062 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: }, 2024-04-02T04:43:58.062 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "pgmap": { 2024-04-02T04:43:58.062 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-02T04:43:58.062 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-02T04:43:58.062 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-02T04:43:58.062 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-02T04:43:58.062 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-02T04:43:58.062 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-02T04:43:58.062 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-02T04:43:58.063 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-02T04:43:58.063 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: }, 2024-04-02T04:43:58.063 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "fsmap": { 2024-04-02T04:43:58.063 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T04:43:58.063 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-02T04:43:58.063 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-02T04:43:58.063 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: }, 2024-04-02T04:43:58.063 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-02T04:43:58.063 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "available": false, 2024-04-02T04:43:58.063 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-02T04:43:58.063 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "modules": [ 2024-04-02T04:43:58.063 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "iostat", 2024-04-02T04:43:58.063 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "nfs", 2024-04-02T04:43:58.063 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "restful" 2024-04-02T04:43:58.064 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: ], 2024-04-02T04:43:58.064 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "services": {} 2024-04-02T04:43:58.064 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: }, 2024-04-02T04:43:58.064 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "servicemap": { 2024-04-02T04:43:58.064 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T04:43:58.064 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "modified": "2024-04-02T04:43:51.159492+0000", 2024-04-02T04:43:58.064 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "services": {} 2024-04-02T04:43:58.064 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: }, 2024-04-02T04:43:58.064 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-02T04:43:58.064 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: } 2024-04-02T04:43:58.315 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:43:58 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/3423005479' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-02T04:43:58.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:43:58 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:43:58.557+0000 7f269ff37000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-02T04:43:58.592 INFO:teuthology.orchestra.run.smithi086.stderr:mgr not available, waiting (1/15)... 2024-04-02T04:43:58.874 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:43:58 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:43:58.646+0000 7f269ff37000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-02T04:43:58.874 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:43:58 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:43:58.727+0000 7f269ff37000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-02T04:43:59.139 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:43:58 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:43:58.870+0000 7f269ff37000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-02T04:43:59.139 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:43:58 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:43:58.950+0000 7f269ff37000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-02T04:43:59.139 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:43:59 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:43:59.055+0000 7f269ff37000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-02T04:43:59.139 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:43:59 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:43:59.135+0000 7f269ff37000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-02T04:43:59.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:43:59 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:43:59.295+0000 7f269ff37000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-02T04:43:59.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:43:59 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:43:59.377+0000 7f269ff37000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-02T04:44:00.715 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:00 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:00.391+0000 7f269ff37000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-02T04:44:00.716 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:00 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:00.519+0000 7f269ff37000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-02T04:44:00.716 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:00 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:00.712+0000 7f269ff37000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-02T04:44:00.984 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:00 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:00.800+0000 7f269ff37000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-02T04:44:00.984 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:00 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:00.943+0000 7f269ff37000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-02T04:44:01.083 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: 2024-04-02T04:44:01.083 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: { 2024-04-02T04:44:01.083 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "fsid": "8862e8fe-f0ab-11ee-b647-cb9ed24678a4", 2024-04-02T04:44:01.083 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "health": { 2024-04-02T04:44:01.083 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-02T04:44:01.083 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "checks": {}, 2024-04-02T04:44:01.083 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "mutes": [] 2024-04-02T04:44:01.083 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: }, 2024-04-02T04:44:01.084 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-02T04:44:01.084 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "quorum": [ 2024-04-02T04:44:01.084 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: 0 2024-04-02T04:44:01.084 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: ], 2024-04-02T04:44:01.084 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-02T04:44:01.084 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "a" 2024-04-02T04:44:01.084 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: ], 2024-04-02T04:44:01.084 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "quorum_age": 5, 2024-04-02T04:44:01.084 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "monmap": { 2024-04-02T04:44:01.084 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T04:44:01.084 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-02T04:44:01.084 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-02T04:44:01.084 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: }, 2024-04-02T04:44:01.084 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "osdmap": { 2024-04-02T04:44:01.085 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T04:44:01.085 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-02T04:44:01.085 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-02T04:44:01.085 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-02T04:44:01.085 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-02T04:44:01.085 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-02T04:44:01.085 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-02T04:44:01.085 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: }, 2024-04-02T04:44:01.085 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "pgmap": { 2024-04-02T04:44:01.085 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-02T04:44:01.085 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-02T04:44:01.085 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-02T04:44:01.085 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-02T04:44:01.085 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-02T04:44:01.086 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-02T04:44:01.086 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-02T04:44:01.086 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-02T04:44:01.086 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: }, 2024-04-02T04:44:01.086 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "fsmap": { 2024-04-02T04:44:01.086 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T04:44:01.086 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-02T04:44:01.086 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-02T04:44:01.086 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: }, 2024-04-02T04:44:01.086 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-02T04:44:01.086 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "available": false, 2024-04-02T04:44:01.086 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-02T04:44:01.086 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "modules": [ 2024-04-02T04:44:01.086 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "iostat", 2024-04-02T04:44:01.086 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "nfs", 2024-04-02T04:44:01.087 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "restful" 2024-04-02T04:44:01.087 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: ], 2024-04-02T04:44:01.087 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "services": {} 2024-04-02T04:44:01.087 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: }, 2024-04-02T04:44:01.087 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "servicemap": { 2024-04-02T04:44:01.087 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T04:44:01.087 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "modified": "2024-04-02T04:43:51.159492+0000", 2024-04-02T04:44:01.087 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "services": {} 2024-04-02T04:44:01.087 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: }, 2024-04-02T04:44:01.087 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-02T04:44:01.087 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: } 2024-04-02T04:44:01.571 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:01 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:01.265+0000 7f269ff37000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-02T04:44:01.571 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:01 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:01.484+0000 7f269ff37000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-02T04:44:01.571 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:01 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:01.568+0000 7f269ff37000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-02T04:44:01.690 INFO:teuthology.orchestra.run.smithi086.stderr:mgr not available, waiting (2/15)... 2024-04-02T04:44:01.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:01 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1524000057' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-02T04:44:01.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:01 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:01.740+0000 7f269ff37000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-02T04:44:02.940 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:02 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:02.647+0000 7f269ff37000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-02T04:44:02.940 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:02 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:02.937+0000 7f269ff37000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-02T04:44:03.234 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:03.030+0000 7f269ff37000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-02T04:44:03.234 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:03.179+0000 7f269ff37000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-02T04:44:03.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:03 smithi086 ceph-mon[24811]: Activating manager daemon y 2024-04-02T04:44:03.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:03 smithi086 ceph-mon[24811]: mgrmap e2: y(active, starting, since 0.00293026s) 2024-04-02T04:44:03.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:03 smithi086 ceph-mon[24811]: from='mgr.14100 172.21.15.86:0/4205062175' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T04:44:03.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:03 smithi086 ceph-mon[24811]: from='mgr.14100 172.21.15.86:0/4205062175' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T04:44:03.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:03 smithi086 ceph-mon[24811]: from='mgr.14100 172.21.15.86:0/4205062175' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T04:44:03.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:03 smithi086 ceph-mon[24811]: from='mgr.14100 172.21.15.86:0/4205062175' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T04:44:03.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:03 smithi086 ceph-mon[24811]: from='mgr.14100 172.21.15.86:0/4205062175' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T04:44:03.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:03 smithi086 ceph-mon[24811]: Manager daemon y is now available 2024-04-02T04:44:03.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:03 smithi086 ceph-mon[24811]: from='mgr.14100 172.21.15.86:0/4205062175' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:44:03.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:03 smithi086 ceph-mon[24811]: from='mgr.14100 172.21.15.86:0/4205062175' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:44:03.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:03 smithi086 ceph-mon[24811]: from='mgr.14100 172.21.15.86:0/4205062175' entity='mgr.y' 2024-04-02T04:44:03.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:03 smithi086 ceph-mon[24811]: from='mgr.14100 172.21.15.86:0/4205062175' entity='mgr.y' 2024-04-02T04:44:03.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:03 smithi086 ceph-mon[24811]: from='mgr.14100 172.21.15.86:0/4205062175' entity='mgr.y' 2024-04-02T04:44:05.470 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:05 smithi086 ceph-mon[24811]: mgrmap e3: y(active, since 1.00694s) 2024-04-02T04:44:06.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:06 smithi086 ceph-mon[24811]: mgrmap e4: y(active, since 2s) 2024-04-02T04:44:07.087 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: 2024-04-02T04:44:07.087 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: { 2024-04-02T04:44:07.087 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "fsid": "8862e8fe-f0ab-11ee-b647-cb9ed24678a4", 2024-04-02T04:44:07.087 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "health": { 2024-04-02T04:44:07.087 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-02T04:44:07.087 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "checks": {}, 2024-04-02T04:44:07.088 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "mutes": [] 2024-04-02T04:44:07.088 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: }, 2024-04-02T04:44:07.088 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-02T04:44:07.088 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "quorum": [ 2024-04-02T04:44:07.088 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: 0 2024-04-02T04:44:07.088 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: ], 2024-04-02T04:44:07.088 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-02T04:44:07.088 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "a" 2024-04-02T04:44:07.088 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: ], 2024-04-02T04:44:07.088 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "quorum_age": 11, 2024-04-02T04:44:07.088 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "monmap": { 2024-04-02T04:44:07.088 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T04:44:07.088 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-02T04:44:07.088 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-02T04:44:07.088 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: }, 2024-04-02T04:44:07.089 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "osdmap": { 2024-04-02T04:44:07.089 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T04:44:07.089 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-02T04:44:07.089 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-02T04:44:07.089 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-02T04:44:07.089 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-02T04:44:07.092 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-02T04:44:07.092 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-02T04:44:07.092 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: }, 2024-04-02T04:44:07.092 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "pgmap": { 2024-04-02T04:44:07.092 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-02T04:44:07.093 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-02T04:44:07.093 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-02T04:44:07.093 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-02T04:44:07.093 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-02T04:44:07.093 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-02T04:44:07.093 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-02T04:44:07.093 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-02T04:44:07.093 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: }, 2024-04-02T04:44:07.093 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "fsmap": { 2024-04-02T04:44:07.093 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T04:44:07.093 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-02T04:44:07.093 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-02T04:44:07.093 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: }, 2024-04-02T04:44:07.093 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-02T04:44:07.094 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "available": true, 2024-04-02T04:44:07.094 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-02T04:44:07.094 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "modules": [ 2024-04-02T04:44:07.094 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "iostat", 2024-04-02T04:44:07.094 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "nfs", 2024-04-02T04:44:07.094 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "restful" 2024-04-02T04:44:07.094 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: ], 2024-04-02T04:44:07.094 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "services": {} 2024-04-02T04:44:07.094 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: }, 2024-04-02T04:44:07.094 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "servicemap": { 2024-04-02T04:44:07.094 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T04:44:07.094 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "modified": "2024-04-02T04:43:51.159492+0000", 2024-04-02T04:44:07.094 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "services": {} 2024-04-02T04:44:07.094 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: }, 2024-04-02T04:44:07.095 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-02T04:44:07.095 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: } 2024-04-02T04:44:07.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:07 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/272325687' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-02T04:44:13.090 INFO:teuthology.orchestra.run.smithi086.stderr:mgr is available 2024-04-02T04:44:15.390 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: 2024-04-02T04:44:15.390 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: [global] 2024-04-02T04:44:15.390 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: fsid = 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 2024-04-02T04:44:15.390 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-02T04:44:15.391 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-02T04:44:15.391 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-02T04:44:15.391 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-02T04:44:15.391 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: 2024-04-02T04:44:15.391 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: [mgr] 2024-04-02T04:44:15.391 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-02T04:44:15.391 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: 2024-04-02T04:44:15.392 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: [osd] 2024-04-02T04:44:15.392 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-02T04:44:15.392 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-02T04:44:15.687 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:15 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/3565469007' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-02T04:44:15.687 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:15 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/3565469007' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-02T04:44:15.947 INFO:teuthology.orchestra.run.smithi086.stderr:Enabling cephadm module... 2024-04-02T04:44:16.943 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:16 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1127659949' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-02T04:44:16.944 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:16 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ignoring --setuser ceph since I am not root 2024-04-02T04:44:16.944 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:16 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ignoring --setgroup ceph since I am not root 2024-04-02T04:44:17.219 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:17 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:17.126+0000 7f4632514000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-02T04:44:17.219 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:17 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:17.216+0000 7f4632514000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-02T04:44:17.828 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:17 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1127659949' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-02T04:44:17.828 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:17 smithi086 ceph-mon[24811]: mgrmap e5: y(active, since 13s) 2024-04-02T04:44:17.869 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: { 2024-04-02T04:44:17.869 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "epoch": 5, 2024-04-02T04:44:17.869 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "available": true, 2024-04-02T04:44:17.869 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "active_name": "y", 2024-04-02T04:44:17.869 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-02T04:44:17.869 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: } 2024-04-02T04:44:18.083 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:17 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:17.824+0000 7f4632514000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-02T04:44:18.083 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:17 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:17.918+0000 7f4632514000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-02T04:44:18.083 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:18 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:18.001+0000 7f4632514000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-02T04:44:18.424 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:18 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:18.151+0000 7f4632514000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-02T04:44:18.425 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:18 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:18.232+0000 7f4632514000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-02T04:44:18.425 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:18 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:18.343+0000 7f4632514000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-02T04:44:18.427 INFO:teuthology.orchestra.run.smithi086.stderr:Waiting for the mgr to restart... 2024-04-02T04:44:18.428 INFO:teuthology.orchestra.run.smithi086.stderr:Waiting for mgr epoch 5... 2024-04-02T04:44:18.689 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:18 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/284847132' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-02T04:44:18.689 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:18 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:18.429+0000 7f4632514000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-02T04:44:18.690 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:18 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:18.598+0000 7f4632514000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-02T04:44:18.690 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:18 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:18.686+0000 7f4632514000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-02T04:44:20.020 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:19 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:19.703+0000 7f4632514000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-02T04:44:20.020 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:19 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:19.830+0000 7f4632514000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-02T04:44:20.020 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:20.016+0000 7f4632514000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-02T04:44:20.299 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:20.103+0000 7f4632514000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-02T04:44:20.300 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:20.239+0000 7f4632514000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-02T04:44:20.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:20.558+0000 7f4632514000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-02T04:44:20.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:20.780+0000 7f4632514000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-02T04:44:21.333 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:20.864+0000 7f4632514000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-02T04:44:21.333 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:21 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:21.032+0000 7f4632514000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-02T04:44:22.237 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:21 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:21.945+0000 7f4632514000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-02T04:44:22.237 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:22 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:22.233+0000 7f4632514000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-02T04:44:22.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:22 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:22.328+0000 7f4632514000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-02T04:44:22.584 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:22 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:22.476+0000 7f4632514000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-02T04:44:23.543 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: { 2024-04-02T04:44:23.543 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-04-02T04:44:23.543 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "initialized": true 2024-04-02T04:44:23.544 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: } 2024-04-02T04:44:23.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:23 smithi086 ceph-mon[24811]: Active manager daemon y restarted 2024-04-02T04:44:23.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:23 smithi086 ceph-mon[24811]: Activating manager daemon y 2024-04-02T04:44:23.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:23 smithi086 ceph-mon[24811]: osdmap e2: 0 total, 0 up, 0 in 2024-04-02T04:44:23.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:23 smithi086 ceph-mon[24811]: mgrmap e6: y(active, starting, since 0.0524003s) 2024-04-02T04:44:23.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:23 smithi086 ceph-mon[24811]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T04:44:23.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:23 smithi086 ceph-mon[24811]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T04:44:23.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:23 smithi086 ceph-mon[24811]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T04:44:23.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:23 smithi086 ceph-mon[24811]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T04:44:23.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:23 smithi086 ceph-mon[24811]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T04:44:23.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:23 smithi086 ceph-mon[24811]: Manager daemon y is now available 2024-04-02T04:44:23.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:23 smithi086 ceph-mon[24811]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' 2024-04-02T04:44:23.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:23 smithi086 ceph-mon[24811]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' 2024-04-02T04:44:23.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:23 smithi086 ceph-mon[24811]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:44:23.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:23 smithi086 ceph-mon[24811]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:44:23.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:23 smithi086 ceph-mon[24811]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:44:23.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:23 smithi086 ceph-mon[24811]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:44:23.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:23 smithi086 ceph-mon[24811]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:44:24.067 INFO:teuthology.orchestra.run.smithi086.stderr:mgr epoch 5 is available 2024-04-02T04:44:24.067 INFO:teuthology.orchestra.run.smithi086.stderr:Setting orchestrator backend to cephadm... 2024-04-02T04:44:24.433 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:24 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: [02/Apr/2024:04:44:24] ENGINE Bus STARTING 2024-04-02T04:44:24.434 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:24 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: [02/Apr/2024:04:44:24] ENGINE Serving on https://172.21.15.86:7150 2024-04-02T04:44:24.434 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:24 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: [02/Apr/2024:04:44:24] ENGINE Bus STARTED 2024-04-02T04:44:24.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:24 smithi086 ceph-mon[24811]: mgrmap e7: y(active, since 1.05487s) 2024-04-02T04:44:24.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:24 smithi086 ceph-mon[24811]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-02T04:44:24.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:24 smithi086 ceph-mon[24811]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-02T04:44:24.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:24 smithi086 ceph-mon[24811]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' 2024-04-02T04:44:24.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:24 smithi086 ceph-mon[24811]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:44:25.778 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:25 smithi086 ceph-mon[24811]: [02/Apr/2024:04:44:24] ENGINE Bus STARTING 2024-04-02T04:44:25.779 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:25 smithi086 ceph-mon[24811]: [02/Apr/2024:04:44:24] ENGINE Serving on https://172.21.15.86:7150 2024-04-02T04:44:25.779 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:25 smithi086 ceph-mon[24811]: [02/Apr/2024:04:44:24] ENGINE Bus STARTED 2024-04-02T04:44:25.779 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:25 smithi086 ceph-mon[24811]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:44:25.779 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:25 smithi086 ceph-mon[24811]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' 2024-04-02T04:44:25.779 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:25 smithi086 ceph-mon[24811]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:44:25.779 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:25 smithi086 ceph-mon[24811]: mgrmap e8: y(active, since 2s) 2024-04-02T04:44:25.779 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: value unchanged 2024-04-02T04:44:26.328 INFO:teuthology.orchestra.run.smithi086.stderr:Generating ssh key... 2024-04-02T04:44:27.043 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:26 smithi086 ceph-mon[24811]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:44:27.297 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:27 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: Generating public/private rsa key pair. 2024-04-02T04:44:27.297 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:27 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: Your identification has been saved in /tmp/tmpctv2efrp/key. 2024-04-02T04:44:27.297 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:27 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: Your public key has been saved in /tmp/tmpctv2efrp/key.pub. 2024-04-02T04:44:27.297 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:27 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: The key fingerprint is: 2024-04-02T04:44:27.298 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:27 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: SHA256:Ic/mprk1GC3ILj3vIjinWdvaS1N75DGyUzMzw3j2akc ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4 2024-04-02T04:44:27.298 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:27 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: The key's randomart image is: 2024-04-02T04:44:27.298 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:27 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: +---[RSA 3072]----+ 2024-04-02T04:44:27.298 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:27 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: | | 2024-04-02T04:44:27.298 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:27 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: | | 2024-04-02T04:44:27.298 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:27 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: | . . | 2024-04-02T04:44:27.298 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:27 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: | . . B . | 2024-04-02T04:44:27.298 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:27 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: | o * S | 2024-04-02T04:44:27.298 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:27 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: | o . / XE | 2024-04-02T04:44:27.302 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:27 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: | .o * = B.. | 2024-04-02T04:44:27.302 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:27 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: |ooo*.+ B.o. | 2024-04-02T04:44:27.302 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:27 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: |o+oo++*o.. | 2024-04-02T04:44:27.302 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:27 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: +----[SHA256]-----+ 2024-04-02T04:44:28.150 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:28 smithi086 ceph-mon[24811]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:44:28.151 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:28 smithi086 ceph-mon[24811]: Generating ssh key... 2024-04-02T04:44:28.151 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:28 smithi086 ceph-mon[24811]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' 2024-04-02T04:44:28.151 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:28 smithi086 ceph-mon[24811]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' 2024-04-02T04:44:28.151 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDhid0MWHW/cAC3ETIP3KnLupFBUaCt/HjY4Ni1ZI/2VUSAB2C2uP4a9p/qmmlKM08ucXLVoKfEV/IWFGvnLC1NpWDgQn7DaHCT0iQPEpwWnZ06vK3SY7AgreJB+7TjTs6yl4K3psxUp/sZJeM6fPx6IsaZDdQ6Arwg4HuSEOMdoDKGMYa1p5OxcxTjCu7fs7oq9Kcl+aJ0K/6umBTpe0f+I1dKM63SNXDdQuPg0KlSY0I8jZTxYszpQiz0pdKqIe5VnwxwWw9JmXxJBmgTBj0AXRECoHh5arxgh5s+zyMOMW6qb89MxHt/yGF8WcbIkwlnEHKdZQ4ePusvF4czfFkjA/DMJjALsVNqMLwhGu8eqCGUg+FsX8YbwJijQdwpS3jQoymxaPR68ld/qegThiclc0JCcSU2kvPpwzRvK6fVqn2DTEGkboZ5R+SbrNjPRUj++wrVEBSW2bWuB/hNFE5LyQbLcpQvC3YvmQqpWMsy/6SZPni271dyfu0WdIEXWGc= ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4 2024-04-02T04:44:28.740 INFO:teuthology.orchestra.run.smithi086.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-04-02T04:44:28.740 INFO:teuthology.orchestra.run.smithi086.stderr:Adding key to root@localhost authorized_keys... 2024-04-02T04:44:28.740 INFO:teuthology.orchestra.run.smithi086.stderr:Adding host smithi086... 2024-04-02T04:44:29.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:29 smithi086 ceph-mon[24811]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:44:30.303 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:30 smithi086 ceph-mon[24811]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi086", "addr": "172.21.15.86", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:44:30.313 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: Added host 'smithi086' with addr '172.21.15.86' 2024-04-02T04:44:30.896 INFO:teuthology.orchestra.run.smithi086.stderr:Deploying unmanaged mon service... 2024-04-02T04:44:31.507 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:31 smithi086 ceph-mon[24811]: Deploying cephadm binary to smithi086 2024-04-02T04:44:31.507 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:31 smithi086 ceph-mon[24811]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' 2024-04-02T04:44:31.507 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:31 smithi086 ceph-mon[24811]: Added host smithi086 2024-04-02T04:44:31.507 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:31 smithi086 ceph-mon[24811]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:44:31.507 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: Scheduled mon update... 2024-04-02T04:44:32.111 INFO:teuthology.orchestra.run.smithi086.stderr:Deploying unmanaged mgr service... 2024-04-02T04:44:32.696 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:32 smithi086 ceph-mon[24811]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:44:32.697 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:32 smithi086 ceph-mon[24811]: Saving service mon spec with placement count:5 2024-04-02T04:44:32.697 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:32 smithi086 ceph-mon[24811]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' 2024-04-02T04:44:32.776 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-04-02T04:44:33.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:33 smithi086 ceph-mon[24811]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:44:33.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:33 smithi086 ceph-mon[24811]: Saving service mgr spec with placement count:2 2024-04-02T04:44:33.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:33 smithi086 ceph-mon[24811]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' 2024-04-02T04:44:33.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:33 smithi086 ceph-mon[24811]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' 2024-04-02T04:44:35.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:35 smithi086 ceph-mon[24811]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' 2024-04-02T04:44:35.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:35 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/2067928772' entity='client.admin' 2024-04-02T04:44:36.582 INFO:teuthology.orchestra.run.smithi086.stderr:Enabling the dashboard module... 2024-04-02T04:44:36.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:36 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/4179686418' entity='client.admin' 2024-04-02T04:44:36.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:36 smithi086 ceph-mon[24811]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' 2024-04-02T04:44:38.404 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:38 smithi086 ceph-mon[24811]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' 2024-04-02T04:44:38.404 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:38 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/690700417' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-02T04:44:38.404 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:38 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ignoring --setuser ceph since I am not root 2024-04-02T04:44:38.404 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:38 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ignoring --setgroup ceph since I am not root 2024-04-02T04:44:38.738 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:38 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:38.644+0000 7f65ac91b000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-02T04:44:38.738 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:38 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:38.734+0000 7f65ac91b000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-02T04:44:39.319 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:39 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/690700417' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-02T04:44:39.319 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:39 smithi086 ceph-mon[24811]: mgrmap e9: y(active, since 15s) 2024-04-02T04:44:39.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:39 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:39.337+0000 7f65ac91b000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-02T04:44:39.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:39 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:39.425+0000 7f65ac91b000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-02T04:44:39.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:39 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:39.511+0000 7f65ac91b000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-02T04:44:39.785 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: { 2024-04-02T04:44:39.786 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "epoch": 9, 2024-04-02T04:44:39.786 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "available": true, 2024-04-02T04:44:39.786 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "active_name": "y", 2024-04-02T04:44:39.786 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-02T04:44:39.786 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: } 2024-04-02T04:44:39.949 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:39 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:39.668+0000 7f65ac91b000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-02T04:44:39.949 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:39 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:39.750+0000 7f65ac91b000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-02T04:44:39.949 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:39 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:39.863+0000 7f65ac91b000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-02T04:44:39.949 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:39 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:39.945+0000 7f65ac91b000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-02T04:44:40.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:40 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/3237345087' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-02T04:44:40.333 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:40 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:40.106+0000 7f65ac91b000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-02T04:44:40.333 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:40 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:40.193+0000 7f65ac91b000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-02T04:44:40.408 INFO:teuthology.orchestra.run.smithi086.stderr:Waiting for the mgr to restart... 2024-04-02T04:44:40.409 INFO:teuthology.orchestra.run.smithi086.stderr:Waiting for mgr epoch 9... 2024-04-02T04:44:41.536 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:41 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:41.219+0000 7f65ac91b000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-02T04:44:41.537 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:41 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:41.347+0000 7f65ac91b000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-02T04:44:41.537 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:41 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:41.533+0000 7f65ac91b000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-02T04:44:41.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:41 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:41.619+0000 7f65ac91b000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-02T04:44:41.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:41 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:41.751+0000 7f65ac91b000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-02T04:44:42.333 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:42.062+0000 7f65ac91b000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-02T04:44:42.333 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:42.278+0000 7f65ac91b000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-02T04:44:42.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:42.362+0000 7f65ac91b000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-02T04:44:42.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:42.529+0000 7f65ac91b000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-02T04:44:43.730 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:43 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:43.435+0000 7f65ac91b000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-02T04:44:43.730 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:43 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:43.727+0000 7f65ac91b000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-02T04:44:43.988 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:43 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:43.825+0000 7f65ac91b000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-02T04:44:44.250 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:43 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:44:43.985+0000 7f65ac91b000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-02T04:44:44.250 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:44 smithi086 ceph-mon[24811]: Active manager daemon y restarted 2024-04-02T04:44:44.250 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:44 smithi086 ceph-mon[24811]: Activating manager daemon y 2024-04-02T04:44:44.250 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:44 smithi086 ceph-mon[24811]: osdmap e3: 0 total, 0 up, 0 in 2024-04-02T04:44:44.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:44 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: [02/Apr/2024:04:44:44] ENGINE Bus STARTING 2024-04-02T04:44:44.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:44 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: [02/Apr/2024:04:44:44] ENGINE Serving on https://172.21.15.86:7150 2024-04-02T04:44:44.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:44:44 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: [02/Apr/2024:04:44:44] ENGINE Bus STARTED 2024-04-02T04:44:45.064 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: { 2024-04-02T04:44:45.064 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-04-02T04:44:45.064 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: "initialized": true 2024-04-02T04:44:45.064 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: } 2024-04-02T04:44:45.177 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:45 smithi086 ceph-mon[24811]: mgrmap e10: y(active, starting, since 0.0555957s) 2024-04-02T04:44:45.178 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:45 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T04:44:45.178 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:45 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T04:44:45.178 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:45 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T04:44:45.178 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:45 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T04:44:45.178 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:45 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T04:44:45.178 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:45 smithi086 ceph-mon[24811]: Manager daemon y is now available 2024-04-02T04:44:45.178 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:45 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:44:45.178 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:45 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:44:45.178 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:45 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:44:45.178 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:45 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:44:45.178 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:45 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:44:45.759 INFO:teuthology.orchestra.run.smithi086.stderr:mgr epoch 9 is available 2024-04-02T04:44:45.759 INFO:teuthology.orchestra.run.smithi086.stderr:Generating a dashboard self-signed certificate... 2024-04-02T04:44:46.265 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:46 smithi086 ceph-mon[24811]: [02/Apr/2024:04:44:44] ENGINE Bus STARTING 2024-04-02T04:44:46.265 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:46 smithi086 ceph-mon[24811]: [02/Apr/2024:04:44:44] ENGINE Serving on https://172.21.15.86:7150 2024-04-02T04:44:46.266 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:46 smithi086 ceph-mon[24811]: [02/Apr/2024:04:44:44] ENGINE Bus STARTED 2024-04-02T04:44:46.266 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:46 smithi086 ceph-mon[24811]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-02T04:44:46.266 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:46 smithi086 ceph-mon[24811]: mgrmap e11: y(active, since 1.06793s) 2024-04-02T04:44:46.266 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:46 smithi086 ceph-mon[24811]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-02T04:44:46.513 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: Self-signed certificate created 2024-04-02T04:44:47.140 INFO:teuthology.orchestra.run.smithi086.stderr:Creating initial admin user... 2024-04-02T04:44:47.406 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:47 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:44:47.406 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:47 smithi086 ceph-mon[24811]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:44:47.406 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:47 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:44:47.406 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:47 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:44:47.407 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:47 smithi086 ceph-mon[24811]: mgrmap e12: y(active, since 2s) 2024-04-02T04:44:47.407 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:47 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:44:48.021 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$2U6xjxmLyYLrYhNLF8J6U.nURVHDcdf8K830fjvs/52G7GPGM.jM.", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1712033088, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-04-02T04:44:48.959 INFO:teuthology.orchestra.run.smithi086.stderr:Fetching dashboard port number... 2024-04-02T04:44:49.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:49 smithi086 ceph-mon[24811]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:44:49.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:49 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:44:50.145 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: 8443 2024-04-02T04:44:50.744 INFO:teuthology.orchestra.run.smithi086.stderr:systemctl: disabled 2024-04-02T04:44:50.750 INFO:teuthology.orchestra.run.smithi086.stderr:systemctl: inactive 2024-04-02T04:44:50.752 INFO:teuthology.orchestra.run.smithi086.stderr:firewalld.service is not enabled 2024-04-02T04:44:50.752 INFO:teuthology.orchestra.run.smithi086.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-04-02T04:44:50.753 INFO:teuthology.orchestra.run.smithi086.stderr:Ceph Dashboard is now available at: 2024-04-02T04:44:50.753 INFO:teuthology.orchestra.run.smithi086.stderr: 2024-04-02T04:44:50.753 INFO:teuthology.orchestra.run.smithi086.stderr: URL: https://smithi086.front.sepia.ceph.com:8443/ 2024-04-02T04:44:50.753 INFO:teuthology.orchestra.run.smithi086.stderr: User: admin 2024-04-02T04:44:50.753 INFO:teuthology.orchestra.run.smithi086.stderr: Password: w3xo59lduo 2024-04-02T04:44:50.754 INFO:teuthology.orchestra.run.smithi086.stderr: 2024-04-02T04:44:50.754 INFO:teuthology.orchestra.run.smithi086.stderr:Enabling autotune for osd_memory_target 2024-04-02T04:44:51.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:51 smithi086 ceph-mon[24811]: mgrmap e13: y(active, since 6s) 2024-04-02T04:44:51.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:51 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/46649572' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-02T04:44:52.700 INFO:teuthology.orchestra.run.smithi086.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-04-02T04:44:53.298 INFO:teuthology.orchestra.run.smithi086.stderr:You can access the Ceph CLI with: 2024-04-02T04:44:53.299 INFO:teuthology.orchestra.run.smithi086.stderr: 2024-04-02T04:44:53.299 INFO:teuthology.orchestra.run.smithi086.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-04-02T04:44:53.299 INFO:teuthology.orchestra.run.smithi086.stderr: 2024-04-02T04:44:53.299 INFO:teuthology.orchestra.run.smithi086.stderr:Please consider enabling telemetry to help improve Ceph: 2024-04-02T04:44:53.299 INFO:teuthology.orchestra.run.smithi086.stderr: 2024-04-02T04:44:53.299 INFO:teuthology.orchestra.run.smithi086.stderr: ceph telemetry on 2024-04-02T04:44:53.299 INFO:teuthology.orchestra.run.smithi086.stderr: 2024-04-02T04:44:53.299 INFO:teuthology.orchestra.run.smithi086.stderr:For more information see: 2024-04-02T04:44:53.299 INFO:teuthology.orchestra.run.smithi086.stderr: 2024-04-02T04:44:53.299 INFO:teuthology.orchestra.run.smithi086.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-04-02T04:44:53.299 INFO:teuthology.orchestra.run.smithi086.stderr: 2024-04-02T04:44:53.299 INFO:teuthology.orchestra.run.smithi086.stderr:Bootstrap complete. 2024-04-02T04:44:53.339 INFO:tasks.cephadm:Fetching config... 2024-04-02T04:44:53.339 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-04-02T04:44:53.339 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-04-02T04:44:53.355 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-04-02T04:44:53.355 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-04-02T04:44:53.355 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-04-02T04:44:53.410 INFO:tasks.cephadm:Fetching mon keyring... 2024-04-02T04:44:53.410 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-04-02T04:44:53.410 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd if=/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/mon.a/keyring of=/dev/stdout 2024-04-02T04:44:53.477 INFO:tasks.cephadm:Fetching pub ssh key... 2024-04-02T04:44:53.477 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-04-02T04:44:53.477 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-04-02T04:44:53.532 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-04-02T04:44:53.533 DEBUG:teuthology.orchestra.run.smithi086:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDhid0MWHW/cAC3ETIP3KnLupFBUaCt/HjY4Ni1ZI/2VUSAB2C2uP4a9p/qmmlKM08ucXLVoKfEV/IWFGvnLC1NpWDgQn7DaHCT0iQPEpwWnZ06vK3SY7AgreJB+7TjTs6yl4K3psxUp/sZJeM6fPx6IsaZDdQ6Arwg4HuSEOMdoDKGMYa1p5OxcxTjCu7fs7oq9Kcl+aJ0K/6umBTpe0f+I1dKM63SNXDdQuPg0KlSY0I8jZTxYszpQiz0pdKqIe5VnwxwWw9JmXxJBmgTBj0AXRECoHh5arxgh5s+zyMOMW6qb89MxHt/yGF8WcbIkwlnEHKdZQ4ePusvF4czfFkjA/DMJjALsVNqMLwhGu8eqCGUg+FsX8YbwJijQdwpS3jQoymxaPR68ld/qegThiclc0JCcSU2kvPpwzRvK6fVqn2DTEGkboZ5R+SbrNjPRUj++wrVEBSW2bWuB/hNFE5LyQbLcpQvC3YvmQqpWMsy/6SZPni271dyfu0WdIEXWGc= ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-02T04:44:53.611 INFO:teuthology.orchestra.run.smithi086.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDhid0MWHW/cAC3ETIP3KnLupFBUaCt/HjY4Ni1ZI/2VUSAB2C2uP4a9p/qmmlKM08ucXLVoKfEV/IWFGvnLC1NpWDgQn7DaHCT0iQPEpwWnZ06vK3SY7AgreJB+7TjTs6yl4K3psxUp/sZJeM6fPx6IsaZDdQ6Arwg4HuSEOMdoDKGMYa1p5OxcxTjCu7fs7oq9Kcl+aJ0K/6umBTpe0f+I1dKM63SNXDdQuPg0KlSY0I8jZTxYszpQiz0pdKqIe5VnwxwWw9JmXxJBmgTBj0AXRECoHh5arxgh5s+zyMOMW6qb89MxHt/yGF8WcbIkwlnEHKdZQ4ePusvF4czfFkjA/DMJjALsVNqMLwhGu8eqCGUg+FsX8YbwJijQdwpS3jQoymxaPR68ld/qegThiclc0JCcSU2kvPpwzRvK6fVqn2DTEGkboZ5R+SbrNjPRUj++wrVEBSW2bWuB/hNFE5LyQbLcpQvC3YvmQqpWMsy/6SZPni271dyfu0WdIEXWGc= ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4 2024-04-02T04:44:53.625 DEBUG:teuthology.orchestra.run.smithi192:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDhid0MWHW/cAC3ETIP3KnLupFBUaCt/HjY4Ni1ZI/2VUSAB2C2uP4a9p/qmmlKM08ucXLVoKfEV/IWFGvnLC1NpWDgQn7DaHCT0iQPEpwWnZ06vK3SY7AgreJB+7TjTs6yl4K3psxUp/sZJeM6fPx6IsaZDdQ6Arwg4HuSEOMdoDKGMYa1p5OxcxTjCu7fs7oq9Kcl+aJ0K/6umBTpe0f+I1dKM63SNXDdQuPg0KlSY0I8jZTxYszpQiz0pdKqIe5VnwxwWw9JmXxJBmgTBj0AXRECoHh5arxgh5s+zyMOMW6qb89MxHt/yGF8WcbIkwlnEHKdZQ4ePusvF4czfFkjA/DMJjALsVNqMLwhGu8eqCGUg+FsX8YbwJijQdwpS3jQoymxaPR68ld/qegThiclc0JCcSU2kvPpwzRvK6fVqn2DTEGkboZ5R+SbrNjPRUj++wrVEBSW2bWuB/hNFE5LyQbLcpQvC3YvmQqpWMsy/6SZPni271dyfu0WdIEXWGc= ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-02T04:44:53.673 INFO:teuthology.orchestra.run.smithi192.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDhid0MWHW/cAC3ETIP3KnLupFBUaCt/HjY4Ni1ZI/2VUSAB2C2uP4a9p/qmmlKM08ucXLVoKfEV/IWFGvnLC1NpWDgQn7DaHCT0iQPEpwWnZ06vK3SY7AgreJB+7TjTs6yl4K3psxUp/sZJeM6fPx6IsaZDdQ6Arwg4HuSEOMdoDKGMYa1p5OxcxTjCu7fs7oq9Kcl+aJ0K/6umBTpe0f+I1dKM63SNXDdQuPg0KlSY0I8jZTxYszpQiz0pdKqIe5VnwxwWw9JmXxJBmgTBj0AXRECoHh5arxgh5s+zyMOMW6qb89MxHt/yGF8WcbIkwlnEHKdZQ4ePusvF4czfFkjA/DMJjALsVNqMLwhGu8eqCGUg+FsX8YbwJijQdwpS3jQoymxaPR68ld/qegThiclc0JCcSU2kvPpwzRvK6fVqn2DTEGkboZ5R+SbrNjPRUj++wrVEBSW2bWuB/hNFE5LyQbLcpQvC3YvmQqpWMsy/6SZPni271dyfu0WdIEXWGc= ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4 2024-04-02T04:44:53.688 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-04-02T04:44:53.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:53 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/3960465699' entity='client.admin' 2024-04-02T04:44:57.567 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-04-02T04:44:57.568 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-04-02T04:44:57.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:57 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:44:57.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:57 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi086", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:44:57.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:57 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:44:57.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:57 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:44:57.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:57 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:44:57.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:57 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/158000671' entity='client.admin' 2024-04-02T04:44:57.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:57 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:44:57.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:44:57 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:00.131 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi192 2024-04-02T04:45:00.131 DEBUG:teuthology.orchestra.run.smithi192:> set -ex 2024-04-02T04:45:00.131 DEBUG:teuthology.orchestra.run.smithi192:> dd of=/etc/ceph/ceph.conf 2024-04-02T04:45:00.150 DEBUG:teuthology.orchestra.run.smithi192:> set -ex 2024-04-02T04:45:00.150 DEBUG:teuthology.orchestra.run.smithi192:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-02T04:45:00.208 INFO:tasks.cephadm:Adding host smithi192 to orchestrator... 2024-04-02T04:45:00.208 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph orch host add smithi192 2024-04-02T04:45:00.618 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:00 smithi086 ceph-mon[24811]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:45:00.618 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:00 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:00.619 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:00 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:45:00.619 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:00 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:00.619 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:00 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:45:00.619 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:00 smithi086 ceph-mon[24811]: Updating smithi086:/etc/ceph/ceph.conf 2024-04-02T04:45:00.619 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:00 smithi086 ceph-mon[24811]: Updating smithi086:/etc/ceph/ceph.client.admin.keyring 2024-04-02T04:45:00.619 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:00 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:00.619 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:00 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:02.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:02 smithi086 ceph-mon[24811]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi192", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:45:02.962 INFO:teuthology.orchestra.run.smithi086.stdout:Added host 'smithi192' with addr '172.21.15.192' 2024-04-02T04:45:03.493 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph orch host ls --format=json 2024-04-02T04:45:04.047 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:03 smithi086 ceph-mon[24811]: Deploying cephadm binary to smithi192 2024-04-02T04:45:04.047 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:03 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:04.047 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:03 smithi086 ceph-mon[24811]: Added host smithi192 2024-04-02T04:45:04.048 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:03 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:45:04.048 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:03 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:04.048 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:03 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:45:04.048 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:03 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:04.048 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:03 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:05.231 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-02T04:45:05.231 INFO:teuthology.orchestra.run.smithi086.stdout:[{"addr": "172.21.15.86", "hostname": "smithi086", "labels": [], "status": ""}, {"addr": "172.21.15.192", "hostname": "smithi192", "labels": [], "status": ""}] 2024-04-02T04:45:05.232 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:05 smithi086 ceph-mon[24811]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:05.232 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:05 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:05.830 INFO:tasks.cephadm:Setting crush tunables to default 2024-04-02T04:45:05.831 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph osd crush tunables default 2024-04-02T04:45:06.804 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:06 smithi086 ceph-mon[24811]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T04:45:06.804 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:06 smithi086 ceph-mon[24811]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:08.354 INFO:teuthology.orchestra.run.smithi086.stderr:adjusted tunables profile to default 2024-04-02T04:45:08.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:08 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:08.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:08 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:08.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:08 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi192", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:45:08.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:08 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:08.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:08 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1872724108' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-02T04:45:08.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:08 smithi086 ceph-mon[24811]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:08.905 INFO:tasks.cephadm:Adding mon.a on smithi086 2024-04-02T04:45:08.905 INFO:tasks.cephadm:Adding mon.c on smithi086 2024-04-02T04:45:08.905 INFO:tasks.cephadm:Adding mon.b on smithi192 2024-04-02T04:45:08.906 DEBUG:teuthology.orchestra.run.smithi192:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph orch apply mon '3;smithi086:172.21.15.86=a;smithi086:[v2:172.21.15.86:3301,v1:172.21.15.86:6790]=c;smithi192:172.21.15.192=b' 2024-04-02T04:45:09.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:09 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1872724108' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-02T04:45:09.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:09 smithi086 ceph-mon[24811]: osdmap e4: 0 total, 0 up, 0 in 2024-04-02T04:45:10.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:10 smithi086 ceph-mon[24811]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:10.838 INFO:teuthology.orchestra.run.smithi192.stdout:Scheduled mon update... 2024-04-02T04:45:11.471 DEBUG:teuthology.orchestra.run.smithi086:mon.c> sudo journalctl -f -n 0 -u ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mon.c.service 2024-04-02T04:45:11.474 DEBUG:teuthology.orchestra.run.smithi192:mon.b> sudo journalctl -f -n 0 -u ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mon.b.service 2024-04-02T04:45:11.477 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-02T04:45:11.477 DEBUG:teuthology.orchestra.run.smithi192:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T04:45:12.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:11 smithi086 ceph-mon[24811]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi086:172.21.15.86=a;smithi086:[v2:172.21.15.86:3301,v1:172.21.15.86:6790]=c;smithi192:172.21.15.192=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:45:12.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:11 smithi086 ceph-mon[24811]: Saving service mon spec with placement smithi086:172.21.15.86=a;smithi086:[v2:172.21.15.86:3301,v1:172.21.15.86:6790]=c;smithi192:172.21.15.192=b;count:3 2024-04-02T04:45:12.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:11 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:12.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:11 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:45:12.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:11 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:12.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:11 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:45:12.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:11 smithi086 ceph-mon[24811]: Updating smithi192:/etc/ceph/ceph.conf 2024-04-02T04:45:12.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:11 smithi086 ceph-mon[24811]: Updating smithi192:/etc/ceph/ceph.client.admin.keyring 2024-04-02T04:45:12.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:11 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:12.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:11 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:12.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:11 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T04:45:12.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:11 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:12.973 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:12 smithi086 ceph-mon[24811]: Deploying daemon mon.c on smithi086 2024-04-02T04:45:12.974 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:12 smithi086 ceph-mon[24811]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:13.218 INFO:teuthology.orchestra.run.smithi192.stdout: 2024-04-02T04:45:13.218 INFO:teuthology.orchestra.run.smithi192.stdout:{"epoch":1,"fsid":"8862e8fe-f0ab-11ee-b647-cb9ed24678a4","modified":"2024-04-02T04:43:48.251359Z","created":"2024-04-02T04:43:48.251359Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:3300","nonce":0},{"type":"v1","addr":"172.21.15.86:6789","nonce":0}]},"addr":"172.21.15.86:6789/0","public_addr":"172.21.15.86:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-02T04:45:13.220 INFO:teuthology.orchestra.run.smithi192.stderr:dumped monmap epoch 1 2024-04-02T04:45:14.231 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:13 smithi086 ceph-mon[24811]: from='client.? 172.21.15.192:0/1176156213' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:45:14.232 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 systemd[1]: Starting Ceph mon.c for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T04:45:14.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 podman[29487]: 2024-04-02 04:45:14.22850555 +0000 UTC m=+0.096612056 container create 49275483a2d4cac9eac2588fcaecffbe3e68069c9b9a80ecebd7d9d6b8c49808 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-c, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, vcs-type=git, maintainer=Guillaume Abrioux , io.openshift.expose-services=, distribution-scope=public, version=8, com.redhat.component=centos-stream-container, name=centos-stream, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.19.8, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, build-date=2022-05-03T08:36:31.336870) 2024-04-02T04:45:14.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 podman[29487]: 2024-04-02 04:45:14.150786804 +0000 UTC m=+0.018893267 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-02T04:45:14.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 podman[29487]: 2024-04-02 04:45:14.372128214 +0000 UTC m=+0.240234678 container init 49275483a2d4cac9eac2588fcaecffbe3e68069c9b9a80ecebd7d9d6b8c49808 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-c, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, release=754) 2024-04-02T04:45:14.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 podman[29487]: 2024-04-02 04:45:14.374938818 +0000 UTC m=+0.243045290 container start 49275483a2d4cac9eac2588fcaecffbe3e68069c9b9a80ecebd7d9d6b8c49808 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-c, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, version=8, RELEASE=HEAD, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., vcs-type=git, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, release=754, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True) 2024-04-02T04:45:14.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: set uid:gid to 167:167 (ceph:ceph) 2024-04-02T04:45:14.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-04-02T04:45:14.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: pidfile_write: ignore empty --pid-file 2024-04-02T04:45:14.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: load: jerasure load: lrc 2024-04-02T04:45:14.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: RocksDB version: 6.15.5 2024-04-02T04:45:14.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-02T04:45:14.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Compile date Apr 18 2022 2024-04-02T04:45:14.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: DB SUMMARY 2024-04-02T04:45:14.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: DB Session ID: 6K7S6W0LDH63UC8ONXQ7 2024-04-02T04:45:14.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: CURRENT file: CURRENT 2024-04-02T04:45:14.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: IDENTITY file: IDENTITY 2024-04-02T04:45:14.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-04-02T04:45:14.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-04-02T04:45:14.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 511 ; 2024-04-02T04:45:14.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.error_if_exists: 0 2024-04-02T04:45:14.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.create_if_missing: 0 2024-04-02T04:45:14.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.paranoid_checks: 1 2024-04-02T04:45:14.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-02T04:45:14.586 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.env: 0x55a43a0c6860 2024-04-02T04:45:14.586 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.fs: Posix File System 2024-04-02T04:45:14.586 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.info_log: 0x55a43c55ddc0 2024-04-02T04:45:14.586 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.max_file_opening_threads: 16 2024-04-02T04:45:14.586 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.statistics: (nil) 2024-04-02T04:45:14.586 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.use_fsync: 0 2024-04-02T04:45:14.588 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.max_log_file_size: 0 2024-04-02T04:45:14.588 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-02T04:45:14.588 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-02T04:45:14.588 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.keep_log_file_num: 1000 2024-04-02T04:45:14.588 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.recycle_log_file_num: 0 2024-04-02T04:45:14.588 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.allow_fallocate: 1 2024-04-02T04:45:14.588 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.allow_mmap_reads: 0 2024-04-02T04:45:14.588 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.allow_mmap_writes: 0 2024-04-02T04:45:14.589 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.use_direct_reads: 0 2024-04-02T04:45:14.589 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-02T04:45:14.589 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.create_missing_column_families: 0 2024-04-02T04:45:14.589 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.db_log_dir: 2024-04-02T04:45:14.589 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2024-04-02T04:45:14.589 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-02T04:45:14.589 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-02T04:45:14.589 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-02T04:45:14.589 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-02T04:45:14.589 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-02T04:45:14.589 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-02T04:45:14.589 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.advise_random_on_open: 1 2024-04-02T04:45:14.589 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.db_write_buffer_size: 0 2024-04-02T04:45:14.589 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.write_buffer_manager: 0x55a43c64e2d0 2024-04-02T04:45:14.589 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-02T04:45:14.589 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-02T04:45:14.590 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-02T04:45:14.590 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-02T04:45:14.590 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.rate_limiter: (nil) 2024-04-02T04:45:14.590 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-02T04:45:14.590 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.wal_recovery_mode: 2 2024-04-02T04:45:14.590 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.enable_thread_tracking: 0 2024-04-02T04:45:14.590 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.enable_pipelined_write: 0 2024-04-02T04:45:14.590 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.unordered_write: 0 2024-04-02T04:45:14.590 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-02T04:45:14.590 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-02T04:45:14.590 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-02T04:45:14.590 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-02T04:45:14.591 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.row_cache: None 2024-04-02T04:45:14.591 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.wal_filter: None 2024-04-02T04:45:14.591 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-02T04:45:14.591 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.allow_ingest_behind: 0 2024-04-02T04:45:14.591 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.preserve_deletes: 0 2024-04-02T04:45:14.591 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.two_write_queues: 0 2024-04-02T04:45:14.591 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.manual_wal_flush: 0 2024-04-02T04:45:14.591 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.atomic_flush: 0 2024-04-02T04:45:14.591 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-02T04:45:14.591 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-02T04:45:14.592 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-02T04:45:14.592 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.log_readahead_size: 0 2024-04-02T04:45:14.592 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-02T04:45:14.592 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.best_efforts_recovery: 0 2024-04-02T04:45:14.592 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-02T04:45:14.592 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-02T04:45:14.592 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.allow_data_in_errors: 0 2024-04-02T04:45:14.592 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.db_host_id: __hostname__ 2024-04-02T04:45:14.592 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.max_background_jobs: 2 2024-04-02T04:45:14.592 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.max_background_compactions: -1 2024-04-02T04:45:14.592 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.max_subcompactions: 1 2024-04-02T04:45:14.592 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-02T04:45:14.592 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-02T04:45:14.592 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-02T04:45:14.593 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.max_total_wal_size: 0 2024-04-02T04:45:14.593 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-02T04:45:14.593 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-02T04:45:14.593 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-02T04:45:14.593 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-02T04:45:14.593 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.max_open_files: -1 2024-04-02T04:45:14.593 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.bytes_per_sync: 0 2024-04-02T04:45:14.593 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-02T04:45:14.593 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-02T04:45:14.593 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.compaction_readahead_size: 0 2024-04-02T04:45:14.593 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.max_background_flushes: -1 2024-04-02T04:45:14.593 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Compression algorithms supported: 2024-04-02T04:45:14.593 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-02T04:45:14.594 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: kZSTD supported: 0 2024-04-02T04:45:14.594 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: kXpressCompression supported: 0 2024-04-02T04:45:14.594 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: kLZ4HCCompression supported: 1 2024-04-02T04:45:14.594 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: kLZ4Compression supported: 1 2024-04-02T04:45:14.594 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: kBZip2Compression supported: 0 2024-04-02T04:45:14.594 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: kZlibCompression supported: 1 2024-04-02T04:45:14.594 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: kSnappyCompression supported: 1 2024-04-02T04:45:14.594 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-02T04:45:14.594 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2024-04-02T04:45:14.594 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-04-02T04:45:14.594 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-02T04:45:14.594 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.merge_operator: 2024-04-02T04:45:14.594 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.compaction_filter: None 2024-04-02T04:45:14.594 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.compaction_filter_factory: None 2024-04-02T04:45:14.594 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.sst_partitioner_factory: None 2024-04-02T04:45:14.595 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-02T04:45:14.595 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-02T04:45:14.595 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55a43c529d00) 2024-04-02T04:45:14.595 INFO:journalctl@ceph.mon.c.smithi086.stdout: cache_index_and_filter_blocks: 1 2024-04-02T04:45:14.595 INFO:journalctl@ceph.mon.c.smithi086.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-02T04:45:14.595 INFO:journalctl@ceph.mon.c.smithi086.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-02T04:45:14.595 INFO:journalctl@ceph.mon.c.smithi086.stdout: pin_top_level_index_and_filter: 1 2024-04-02T04:45:14.595 INFO:journalctl@ceph.mon.c.smithi086.stdout: index_type: 0 2024-04-02T04:45:14.595 INFO:journalctl@ceph.mon.c.smithi086.stdout: data_block_index_type: 0 2024-04-02T04:45:14.595 INFO:journalctl@ceph.mon.c.smithi086.stdout: index_shortening: 1 2024-04-02T04:45:14.595 INFO:journalctl@ceph.mon.c.smithi086.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-02T04:45:14.595 INFO:journalctl@ceph.mon.c.smithi086.stdout: hash_index_allow_collision: 1 2024-04-02T04:45:14.595 INFO:journalctl@ceph.mon.c.smithi086.stdout: checksum: 1 2024-04-02T04:45:14.596 INFO:journalctl@ceph.mon.c.smithi086.stdout: no_block_cache: 0 2024-04-02T04:45:14.596 INFO:journalctl@ceph.mon.c.smithi086.stdout: block_cache: 0x55a43c594170 2024-04-02T04:45:14.596 INFO:journalctl@ceph.mon.c.smithi086.stdout: block_cache_name: BinnedLRUCache 2024-04-02T04:45:14.596 INFO:journalctl@ceph.mon.c.smithi086.stdout: block_cache_options: 2024-04-02T04:45:14.596 INFO:journalctl@ceph.mon.c.smithi086.stdout: capacity : 536870912 2024-04-02T04:45:14.596 INFO:journalctl@ceph.mon.c.smithi086.stdout: num_shard_bits : 4 2024-04-02T04:45:14.596 INFO:journalctl@ceph.mon.c.smithi086.stdout: strict_capacity_limit : 0 2024-04-02T04:45:14.596 INFO:journalctl@ceph.mon.c.smithi086.stdout: high_pri_pool_ratio: 0.000 2024-04-02T04:45:14.596 INFO:journalctl@ceph.mon.c.smithi086.stdout: block_cache_compressed: (nil) 2024-04-02T04:45:14.596 INFO:journalctl@ceph.mon.c.smithi086.stdout: persistent_cache: (nil) 2024-04-02T04:45:14.596 INFO:journalctl@ceph.mon.c.smithi086.stdout: block_size: 4096 2024-04-02T04:45:14.596 INFO:journalctl@ceph.mon.c.smithi086.stdout: block_size_deviation: 10 2024-04-02T04:45:14.596 INFO:journalctl@ceph.mon.c.smithi086.stdout: block_restart_interval: 16 2024-04-02T04:45:14.596 INFO:journalctl@ceph.mon.c.smithi086.stdout: index_block_restart_interval: 1 2024-04-02T04:45:14.596 INFO:journalctl@ceph.mon.c.smithi086.stdout: metadata_block_size: 4096 2024-04-02T04:45:14.596 INFO:journalctl@ceph.mon.c.smithi086.stdout: partition_filters: 0 2024-04-02T04:45:14.597 INFO:journalctl@ceph.mon.c.smithi086.stdout: use_delta_encoding: 1 2024-04-02T04:45:14.597 INFO:journalctl@ceph.mon.c.smithi086.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-02T04:45:14.597 INFO:journalctl@ceph.mon.c.smithi086.stdout: whole_key_filtering: 1 2024-04-02T04:45:14.597 INFO:journalctl@ceph.mon.c.smithi086.stdout: verify_compression: 0 2024-04-02T04:45:14.597 INFO:journalctl@ceph.mon.c.smithi086.stdout: read_amp_bytes_per_bit: 0 2024-04-02T04:45:14.597 INFO:journalctl@ceph.mon.c.smithi086.stdout: format_version: 4 2024-04-02T04:45:14.597 INFO:journalctl@ceph.mon.c.smithi086.stdout: enable_index_compression: 1 2024-04-02T04:45:14.597 INFO:journalctl@ceph.mon.c.smithi086.stdout: block_align: 0 2024-04-02T04:45:14.597 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.write_buffer_size: 33554432 2024-04-02T04:45:14.597 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.max_write_buffer_number: 2 2024-04-02T04:45:14.597 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.compression: NoCompression 2024-04-02T04:45:14.597 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.bottommost_compression: Disabled 2024-04-02T04:45:14.597 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.prefix_extractor: nullptr 2024-04-02T04:45:14.597 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-02T04:45:14.598 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.num_levels: 7 2024-04-02T04:45:14.598 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-02T04:45:14.598 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-02T04:45:14.598 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-02T04:45:14.598 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-02T04:45:14.598 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-02T04:45:14.598 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-02T04:45:14.598 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-02T04:45:14.598 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-02T04:45:14.598 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-02T04:45:14.598 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-02T04:45:14.598 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-02T04:45:14.598 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.compression_opts.level: 32767 2024-04-02T04:45:14.599 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.compression_opts.strategy: 0 2024-04-02T04:45:14.599 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-02T04:45:14.599 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-02T04:45:14.599 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-02T04:45:14.599 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.compression_opts.enabled: false 2024-04-02T04:45:14.599 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-02T04:45:14.599 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-02T04:45:14.599 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-02T04:45:14.599 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.target_file_size_base: 67108864 2024-04-02T04:45:14.599 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-02T04:45:14.599 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-02T04:45:14.599 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-02T04:45:14.599 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-02T04:45:14.600 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-02T04:45:14.600 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-02T04:45:14.600 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-02T04:45:14.600 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-02T04:45:14.600 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-02T04:45:14.600 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-02T04:45:14.600 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-02T04:45:14.600 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-02T04:45:14.600 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-02T04:45:14.600 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.arena_block_size: 4194304 2024-04-02T04:45:14.600 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-02T04:45:14.600 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-02T04:45:14.600 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-02T04:45:14.600 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.disable_auto_compactions: 0 2024-04-02T04:45:14.600 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-02T04:45:14.600 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-02T04:45:14.601 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-02T04:45:14.601 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-02T04:45:14.601 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-02T04:45:14.601 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-02T04:45:14.601 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-02T04:45:14.601 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-02T04:45:14.601 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-02T04:45:14.601 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-02T04:45:14.601 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.table_properties_collectors: 2024-04-02T04:45:14.601 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.inplace_update_support: 0 2024-04-02T04:45:14.601 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-02T04:45:14.601 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-02T04:45:14.601 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-02T04:45:14.601 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-02T04:45:14.602 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.bloom_locality: 0 2024-04-02T04:45:14.602 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.max_successive_merges: 0 2024-04-02T04:45:14.602 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-02T04:45:14.602 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.paranoid_file_checks: 0 2024-04-02T04:45:14.602 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.force_consistency_checks: 1 2024-04-02T04:45:14.602 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.report_bg_io_stats: 0 2024-04-02T04:45:14.602 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.ttl: 2592000 2024-04-02T04:45:14.602 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-02T04:45:14.602 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.enable_blob_files: false 2024-04-02T04:45:14.602 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.min_blob_size: 0 2024-04-02T04:45:14.602 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.blob_file_size: 268435456 2024-04-02T04:45:14.602 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-02T04:45:14.602 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-02T04:45:14.603 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-02T04:45:14.603 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-02T04:45:14.603 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-04-02T04:45:14.603 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-04-02T04:45:14.603 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712033114400661, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-04-02T04:45:14.603 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-04-02T04:45:14.603 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-02T04:45:14.603 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712033114400979, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1712033114, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "d9e61803-5eb7-49dc-978e-698a26923dc8", "db_session_id": "6K7S6W0LDH63UC8ONXQ7"}} 2024-04-02T04:45:14.603 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-04-02T04:45:14.603 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712033114401309, "job": 1, "event": "recovery_finished"} 2024-04-02T04:45:14.603 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-02T04:45:14.603 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55a43c57aa80 2024-04-02T04:45:14.603 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: DB pointer 0x55a43c58a000 2024-04-02T04:45:14.603 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-04-02T04:45:14.604 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-02T04:45:14.604 INFO:journalctl@ceph.mon.c.smithi086.stdout: ** DB Stats ** 2024-04-02T04:45:14.604 INFO:journalctl@ceph.mon.c.smithi086.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-02T04:45:14.604 INFO:journalctl@ceph.mon.c.smithi086.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-02T04:45:14.604 INFO:journalctl@ceph.mon.c.smithi086.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-02T04:45:14.604 INFO:journalctl@ceph.mon.c.smithi086.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-02T04:45:14.604 INFO:journalctl@ceph.mon.c.smithi086.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-02T04:45:14.604 INFO:journalctl@ceph.mon.c.smithi086.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-02T04:45:14.604 INFO:journalctl@ceph.mon.c.smithi086.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-02T04:45:14.604 INFO:journalctl@ceph.mon.c.smithi086.stdout: 2024-04-02T04:45:14.604 INFO:journalctl@ceph.mon.c.smithi086.stdout: ** Compaction Stats [default] ** 2024-04-02T04:45:14.604 INFO:journalctl@ceph.mon.c.smithi086.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-02T04:45:14.604 INFO:journalctl@ceph.mon.c.smithi086.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T04:45:14.605 INFO:journalctl@ceph.mon.c.smithi086.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2024-04-02T04:45:14.605 INFO:journalctl@ceph.mon.c.smithi086.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2024-04-02T04:45:14.605 INFO:journalctl@ceph.mon.c.smithi086.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2024-04-02T04:45:14.605 INFO:journalctl@ceph.mon.c.smithi086.stdout: 2024-04-02T04:45:14.605 INFO:journalctl@ceph.mon.c.smithi086.stdout: ** Compaction Stats [default] ** 2024-04-02T04:45:14.605 INFO:journalctl@ceph.mon.c.smithi086.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-02T04:45:14.605 INFO:journalctl@ceph.mon.c.smithi086.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T04:45:14.605 INFO:journalctl@ceph.mon.c.smithi086.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2024-04-02T04:45:14.605 INFO:journalctl@ceph.mon.c.smithi086.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-02T04:45:14.605 INFO:journalctl@ceph.mon.c.smithi086.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-02T04:45:14.605 INFO:journalctl@ceph.mon.c.smithi086.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-02T04:45:14.605 INFO:journalctl@ceph.mon.c.smithi086.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-02T04:45:14.606 INFO:journalctl@ceph.mon.c.smithi086.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-02T04:45:14.606 INFO:journalctl@ceph.mon.c.smithi086.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-02T04:45:14.606 INFO:journalctl@ceph.mon.c.smithi086.stdout: Cumulative compaction: 0.00 GB write, 0.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T04:45:14.606 INFO:journalctl@ceph.mon.c.smithi086.stdout: Interval compaction: 0.00 GB write, 0.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T04:45:14.606 INFO:journalctl@ceph.mon.c.smithi086.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-02T04:45:14.606 INFO:journalctl@ceph.mon.c.smithi086.stdout: 2024-04-02T04:45:14.606 INFO:journalctl@ceph.mon.c.smithi086.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-02T04:45:14.606 INFO:journalctl@ceph.mon.c.smithi086.stdout: 2024-04-02T04:45:14.606 INFO:journalctl@ceph.mon.c.smithi086.stdout: ** Compaction Stats [default] ** 2024-04-02T04:45:14.606 INFO:journalctl@ceph.mon.c.smithi086.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-02T04:45:14.606 INFO:journalctl@ceph.mon.c.smithi086.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T04:45:14.606 INFO:journalctl@ceph.mon.c.smithi086.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2024-04-02T04:45:14.606 INFO:journalctl@ceph.mon.c.smithi086.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2024-04-02T04:45:14.607 INFO:journalctl@ceph.mon.c.smithi086.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-02T04:45:14.607 INFO:journalctl@ceph.mon.c.smithi086.stdout: 2024-04-02T04:45:14.607 INFO:journalctl@ceph.mon.c.smithi086.stdout: ** Compaction Stats [default] ** 2024-04-02T04:45:14.607 INFO:journalctl@ceph.mon.c.smithi086.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-02T04:45:14.607 INFO:journalctl@ceph.mon.c.smithi086.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T04:45:14.607 INFO:journalctl@ceph.mon.c.smithi086.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2024-04-02T04:45:14.607 INFO:journalctl@ceph.mon.c.smithi086.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-02T04:45:14.607 INFO:journalctl@ceph.mon.c.smithi086.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-02T04:45:14.607 INFO:journalctl@ceph.mon.c.smithi086.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-02T04:45:14.607 INFO:journalctl@ceph.mon.c.smithi086.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-02T04:45:14.607 INFO:journalctl@ceph.mon.c.smithi086.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-02T04:45:14.607 INFO:journalctl@ceph.mon.c.smithi086.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-02T04:45:14.607 INFO:journalctl@ceph.mon.c.smithi086.stdout: Cumulative compaction: 0.00 GB write, 0.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T04:45:14.607 INFO:journalctl@ceph.mon.c.smithi086.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T04:45:14.608 INFO:journalctl@ceph.mon.c.smithi086.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-02T04:45:14.608 INFO:journalctl@ceph.mon.c.smithi086.stdout: 2024-04-02T04:45:14.608 INFO:journalctl@ceph.mon.c.smithi086.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-02T04:45:14.608 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-04-02T04:45:14.608 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: using public_addrv [v2:172.21.15.86:3301/0,v1:172.21.15.86:6790/0] 2024-04-02T04:45:14.608 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: starting mon.c rank -1 at public addrs [v2:172.21.15.86:3301/0,v1:172.21.15.86:6790/0] at bind addrs [v2:172.21.15.86:3301/0,v1:172.21.15.86:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 2024-04-02T04:45:14.608 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: mon.c@-1(???) e0 preinit fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 2024-04-02T04:45:14.608 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: mon.c@-1(synchronizing).mds e1 new map 2024-04-02T04:45:14.608 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: mon.c@-1(synchronizing).mds e1 print_map 2024-04-02T04:45:14.608 INFO:journalctl@ceph.mon.c.smithi086.stdout: e1 2024-04-02T04:45:14.608 INFO:journalctl@ceph.mon.c.smithi086.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-02T04:45:14.608 INFO:journalctl@ceph.mon.c.smithi086.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-02T04:45:14.608 INFO:journalctl@ceph.mon.c.smithi086.stdout: legacy client fscid: -1 2024-04-02T04:45:14.608 INFO:journalctl@ceph.mon.c.smithi086.stdout: 2024-04-02T04:45:14.609 INFO:journalctl@ceph.mon.c.smithi086.stdout: No filesystems configured 2024-04-02T04:45:14.609 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-04-02T04:45:14.609 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-04-02T04:45:14.609 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-04-02T04:45:14.609 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-04-02T04:45:14.609 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-04-02T04:45:14.609 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-04-02T04:45:14.609 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-02T04:45:14.609 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-02T04:45:14.609 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-02T04:45:14.609 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-02T04:45:14.609 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: mkfs 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 2024-04-02T04:45:14.609 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-02T04:45:14.610 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-02T04:45:14.610 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: monmap e1: 1 mons at {a=[v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0]} 2024-04-02T04:45:14.610 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: fsmap 2024-04-02T04:45:14.610 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: osdmap e1: 0 total, 0 up, 0 in 2024-04-02T04:45:14.610 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: mgrmap e1: no daemons active 2024-04-02T04:45:14.610 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/3437553354' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-02T04:45:14.610 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/3334165407' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-02T04:45:14.610 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/3334165407' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-02T04:45:14.610 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/4286765569' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:14.610 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-02T04:45:14.610 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: monmap e1: 1 mons at {a=[v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0]} 2024-04-02T04:45:14.610 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: fsmap 2024-04-02T04:45:14.611 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: osdmap e1: 0 total, 0 up, 0 in 2024-04-02T04:45:14.611 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: mgrmap e1: no daemons active 2024-04-02T04:45:14.611 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/494209034' entity='client.admin' 2024-04-02T04:45:14.611 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/3423005479' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-02T04:45:14.611 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1524000057' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-02T04:45:14.611 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: Activating manager daemon y 2024-04-02T04:45:14.611 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: mgrmap e2: y(active, starting, since 0.00293026s) 2024-04-02T04:45:14.611 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14100 172.21.15.86:0/4205062175' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T04:45:14.611 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14100 172.21.15.86:0/4205062175' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T04:45:14.611 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14100 172.21.15.86:0/4205062175' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T04:45:14.611 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14100 172.21.15.86:0/4205062175' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T04:45:14.611 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14100 172.21.15.86:0/4205062175' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T04:45:14.611 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: Manager daemon y is now available 2024-04-02T04:45:14.611 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14100 172.21.15.86:0/4205062175' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:45:14.611 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14100 172.21.15.86:0/4205062175' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:45:14.612 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14100 172.21.15.86:0/4205062175' entity='mgr.y' 2024-04-02T04:45:14.612 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14100 172.21.15.86:0/4205062175' entity='mgr.y' 2024-04-02T04:45:14.612 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14100 172.21.15.86:0/4205062175' entity='mgr.y' 2024-04-02T04:45:14.612 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: mgrmap e3: y(active, since 1.00694s) 2024-04-02T04:45:14.612 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: mgrmap e4: y(active, since 2s) 2024-04-02T04:45:14.612 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/272325687' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-02T04:45:14.612 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/3565469007' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-02T04:45:14.612 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/3565469007' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-02T04:45:14.612 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1127659949' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-02T04:45:14.612 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1127659949' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-02T04:45:14.612 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: mgrmap e5: y(active, since 13s) 2024-04-02T04:45:14.612 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/284847132' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-02T04:45:14.612 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: Active manager daemon y restarted 2024-04-02T04:45:14.613 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: Activating manager daemon y 2024-04-02T04:45:14.613 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: osdmap e2: 0 total, 0 up, 0 in 2024-04-02T04:45:14.613 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: mgrmap e6: y(active, starting, since 0.0524003s) 2024-04-02T04:45:14.613 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T04:45:14.613 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T04:45:14.613 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T04:45:14.613 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T04:45:14.613 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T04:45:14.613 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: Manager daemon y is now available 2024-04-02T04:45:14.613 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' 2024-04-02T04:45:14.613 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' 2024-04-02T04:45:14.613 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:45:14.613 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:45:14.614 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:45:14.614 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:45:14.614 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:45:14.614 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: mgrmap e7: y(active, since 1.05487s) 2024-04-02T04:45:14.614 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-02T04:45:14.614 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-02T04:45:14.614 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' 2024-04-02T04:45:14.614 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:45:14.614 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: [02/Apr/2024:04:44:24] ENGINE Bus STARTING 2024-04-02T04:45:14.614 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: [02/Apr/2024:04:44:24] ENGINE Serving on https://172.21.15.86:7150 2024-04-02T04:45:14.614 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: [02/Apr/2024:04:44:24] ENGINE Bus STARTED 2024-04-02T04:45:14.614 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:45:14.614 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' 2024-04-02T04:45:14.615 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:45:14.615 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: mgrmap e8: y(active, since 2s) 2024-04-02T04:45:14.615 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:45:14.615 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:45:14.615 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: Generating ssh key... 2024-04-02T04:45:14.615 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' 2024-04-02T04:45:14.615 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' 2024-04-02T04:45:14.615 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:45:14.615 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi086", "addr": "172.21.15.86", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:45:14.615 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: Deploying cephadm binary to smithi086 2024-04-02T04:45:14.615 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' 2024-04-02T04:45:14.615 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: Added host smithi086 2024-04-02T04:45:14.615 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:45:14.616 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:45:14.616 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: Saving service mon spec with placement count:5 2024-04-02T04:45:14.616 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' 2024-04-02T04:45:14.616 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:45:14.616 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: Saving service mgr spec with placement count:2 2024-04-02T04:45:14.616 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' 2024-04-02T04:45:14.616 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' 2024-04-02T04:45:14.616 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' 2024-04-02T04:45:14.616 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/2067928772' entity='client.admin' 2024-04-02T04:45:14.616 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/4179686418' entity='client.admin' 2024-04-02T04:45:14.616 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' 2024-04-02T04:45:14.616 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14118 172.21.15.86:0/2706402651' entity='mgr.y' 2024-04-02T04:45:14.616 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/690700417' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-02T04:45:14.616 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/690700417' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-02T04:45:14.617 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: mgrmap e9: y(active, since 15s) 2024-04-02T04:45:14.617 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/3237345087' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-02T04:45:14.617 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: Active manager daemon y restarted 2024-04-02T04:45:14.617 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: Activating manager daemon y 2024-04-02T04:45:14.617 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: osdmap e3: 0 total, 0 up, 0 in 2024-04-02T04:45:14.617 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: mgrmap e10: y(active, starting, since 0.0555957s) 2024-04-02T04:45:14.617 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T04:45:14.617 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T04:45:14.617 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T04:45:14.617 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T04:45:14.617 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T04:45:14.617 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: Manager daemon y is now available 2024-04-02T04:45:14.617 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:45:14.617 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:45:14.618 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:45:14.618 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:45:14.618 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:14.618 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: [02/Apr/2024:04:44:44] ENGINE Bus STARTING 2024-04-02T04:45:14.618 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: [02/Apr/2024:04:44:44] ENGINE Serving on https://172.21.15.86:7150 2024-04-02T04:45:14.618 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: [02/Apr/2024:04:44:44] ENGINE Bus STARTED 2024-04-02T04:45:14.618 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-02T04:45:14.618 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: mgrmap e11: y(active, since 1.06793s) 2024-04-02T04:45:14.618 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-02T04:45:14.618 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:14.618 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:45:14.618 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:14.618 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:14.618 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: mgrmap e12: y(active, since 2s) 2024-04-02T04:45:14.619 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:14.619 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:45:14.619 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:14.619 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: mgrmap e13: y(active, since 6s) 2024-04-02T04:45:14.619 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/46649572' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-02T04:45:14.619 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/3960465699' entity='client.admin' 2024-04-02T04:45:14.619 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:14.619 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi086", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:45:14.619 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:14.619 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:45:14.620 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:14.620 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/158000671' entity='client.admin' 2024-04-02T04:45:14.620 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:45:14.620 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:14.620 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:45:14.620 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:14.620 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:45:14.620 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:14.620 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:45:14.620 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: Updating smithi086:/etc/ceph/ceph.conf 2024-04-02T04:45:14.620 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: Updating smithi086:/etc/ceph/ceph.client.admin.keyring 2024-04-02T04:45:14.620 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:14.620 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:14.620 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi192", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:45:14.620 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: Deploying cephadm binary to smithi192 2024-04-02T04:45:14.621 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:14.621 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: Added host smithi192 2024-04-02T04:45:14.621 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:45:14.621 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:14.621 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:45:14.621 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:14.621 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:14.621 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:14.621 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:14.621 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T04:45:14.621 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:14.621 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:14.621 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:14.621 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi192", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:45:14.622 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:14.622 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1872724108' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-02T04:45:14.622 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:14.622 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1872724108' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-02T04:45:14.622 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: osdmap e4: 0 total, 0 up, 0 in 2024-04-02T04:45:14.622 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:14.622 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi086:172.21.15.86=a;smithi086:[v2:172.21.15.86:3301,v1:172.21.15.86:6790]=c;smithi192:172.21.15.192=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:45:14.622 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: Saving service mon spec with placement smithi086:172.21.15.86=a;smithi086:[v2:172.21.15.86:3301,v1:172.21.15.86:6790]=c;smithi192:172.21.15.192=b;count:3 2024-04-02T04:45:14.622 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:14.622 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:45:14.622 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:14.622 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:45:14.622 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: Updating smithi192:/etc/ceph/ceph.conf 2024-04-02T04:45:14.623 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: Updating smithi192:/etc/ceph/ceph.client.admin.keyring 2024-04-02T04:45:14.623 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:14.623 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:14.623 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T04:45:14.623 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:14.623 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: Deploying daemon mon.c on smithi086 2024-04-02T04:45:14.623 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:14.623 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: from='client.? 172.21.15.192:0/1176156213' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:45:14.623 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-04-02T04:45:14.623 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 ceph-mon[29505]: mon.c@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-04-02T04:45:14.623 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 bash[29487]: 49275483a2d4cac9eac2588fcaecffbe3e68069c9b9a80ecebd7d9d6b8c49808 2024-04-02T04:45:14.623 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:14 smithi086 systemd[1]: Started Ceph mon.c for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T04:45:14.761 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-02T04:45:14.761 DEBUG:teuthology.orchestra.run.smithi192:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T04:45:19.725 INFO:teuthology.orchestra.run.smithi192.stdout: 2024-04-02T04:45:19.725 INFO:teuthology.orchestra.run.smithi192.stdout:{"epoch":2,"fsid":"8862e8fe-f0ab-11ee-b647-cb9ed24678a4","modified":"2024-04-02T04:45:14.419803Z","created":"2024-04-02T04:43:48.251359Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:3300","nonce":0},{"type":"v1","addr":"172.21.15.86:6789","nonce":0}]},"addr":"172.21.15.86:6789/0","public_addr":"172.21.15.86:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:3301","nonce":0},{"type":"v1","addr":"172.21.15.86:6790","nonce":0}]},"addr":"172.21.15.86:6790/0","public_addr":"172.21.15.86:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-02T04:45:19.727 INFO:teuthology.orchestra.run.smithi192.stderr:dumped monmap epoch 2 2024-04-02T04:45:19.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T04:45:19.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[29505]: mon.a calling monitor election 2024-04-02T04:45:19.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T04:45:19.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T04:45:19.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[29505]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:19.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T04:45:19.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[29505]: mon.c calling monitor election 2024-04-02T04:45:19.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T04:45:19.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[29505]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:19.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T04:45:19.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T04:45:19.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[29505]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-02T04:45:19.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[29505]: monmap e2: 2 mons at {a=[v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0],c=[v2:172.21.15.86:3301/0,v1:172.21.15.86:6790/0]} 2024-04-02T04:45:19.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[29505]: fsmap 2024-04-02T04:45:19.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[29505]: osdmap e4: 0 total, 0 up, 0 in 2024-04-02T04:45:19.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[29505]: mgrmap e13: y(active, since 35s) 2024-04-02T04:45:19.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[29505]: overall HEALTH_OK 2024-04-02T04:45:19.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:19.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T04:45:19.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:19.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T04:45:19.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[24811]: mon.a calling monitor election 2024-04-02T04:45:19.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T04:45:19.836 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T04:45:19.836 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[24811]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:19.836 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T04:45:19.836 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[24811]: mon.c calling monitor election 2024-04-02T04:45:19.836 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T04:45:19.836 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[24811]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:19.836 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T04:45:19.836 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T04:45:19.836 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[24811]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-02T04:45:19.836 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[24811]: monmap e2: 2 mons at {a=[v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0],c=[v2:172.21.15.86:3301/0,v1:172.21.15.86:6790/0]} 2024-04-02T04:45:19.836 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[24811]: fsmap 2024-04-02T04:45:19.836 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[24811]: osdmap e4: 0 total, 0 up, 0 in 2024-04-02T04:45:19.836 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[24811]: mgrmap e13: y(active, since 35s) 2024-04-02T04:45:19.837 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[24811]: overall HEALTH_OK 2024-04-02T04:45:19.837 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:19.837 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T04:45:19.837 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:19 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:20.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:20 smithi086 ceph-mon[29505]: Deploying daemon mon.b on smithi192 2024-04-02T04:45:20.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:20 smithi086 ceph-mon[29505]: from='client.? 172.21.15.192:0/950485049' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:45:20.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:20 smithi086 ceph-mon[29505]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:20.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:20 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T04:45:20.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:20 smithi086 ceph-mon[24811]: Deploying daemon mon.b on smithi192 2024-04-02T04:45:20.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:20 smithi086 ceph-mon[24811]: from='client.? 172.21.15.192:0/950485049' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:45:20.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:20 smithi086 ceph-mon[24811]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:20.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:20 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T04:45:21.262 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-02T04:45:21.262 DEBUG:teuthology.orchestra.run.smithi192:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T04:45:23.454 INFO:teuthology.orchestra.run.smithi192.stdout: 2024-04-02T04:45:23.455 INFO:teuthology.orchestra.run.smithi192.stdout:{"epoch":2,"fsid":"8862e8fe-f0ab-11ee-b647-cb9ed24678a4","modified":"2024-04-02T04:45:14.419803Z","created":"2024-04-02T04:43:48.251359Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:3300","nonce":0},{"type":"v1","addr":"172.21.15.86:6789","nonce":0}]},"addr":"172.21.15.86:6789/0","public_addr":"172.21.15.86:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:3301","nonce":0},{"type":"v1","addr":"172.21.15.86:6790","nonce":0}]},"addr":"172.21.15.86:6790/0","public_addr":"172.21.15.86:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-02T04:45:23.458 INFO:teuthology.orchestra.run.smithi192.stderr:dumped monmap epoch 2 2024-04-02T04:45:23.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:23 smithi086 ceph-mon[24811]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:23.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:23 smithi086 ceph-mon[29505]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:24.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:24 smithi086 ceph-mon[29505]: from='client.? 172.21.15.192:0/1839819296' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:45:24.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:24 smithi086 ceph-mon[24811]: from='client.? 172.21.15.192:0/1839819296' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:45:25.116 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-02T04:45:25.116 DEBUG:teuthology.orchestra.run.smithi192:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T04:45:30.089 INFO:teuthology.orchestra.run.smithi192.stdout: 2024-04-02T04:45:30.090 INFO:teuthology.orchestra.run.smithi192.stdout:{"epoch":3,"fsid":"8862e8fe-f0ab-11ee-b647-cb9ed24678a4","modified":"2024-04-02T04:45:24.807859Z","created":"2024-04-02T04:43:48.251359Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:3300","nonce":0},{"type":"v1","addr":"172.21.15.86:6789","nonce":0}]},"addr":"172.21.15.86:6789/0","public_addr":"172.21.15.86:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:3301","nonce":0},{"type":"v1","addr":"172.21.15.86:6790","nonce":0}]},"addr":"172.21.15.86:6790/0","public_addr":"172.21.15.86:6790/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:3300","nonce":0},{"type":"v1","addr":"172.21.15.192:6789","nonce":0}]},"addr":"172.21.15.192:6789/0","public_addr":"172.21.15.192:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-04-02T04:45:30.091 INFO:teuthology.orchestra.run.smithi192.stderr:dumped monmap epoch 3 2024-04-02T04:45:30.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T04:45:30.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[24811]: mon.a calling monitor election 2024-04-02T04:45:30.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[24811]: mon.c calling monitor election 2024-04-02T04:45:30.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T04:45:30.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T04:45:30.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T04:45:30.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[24811]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:30.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T04:45:30.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[24811]: mon.b calling monitor election 2024-04-02T04:45:30.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T04:45:30.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[24811]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:30.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T04:45:30.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T04:45:30.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[24811]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-02T04:45:30.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[24811]: monmap e3: 3 mons at {a=[v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0],b=[v2:172.21.15.192:3300/0,v1:172.21.15.192:6789/0],c=[v2:172.21.15.86:3301/0,v1:172.21.15.86:6790/0]} 2024-04-02T04:45:30.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[24811]: fsmap 2024-04-02T04:45:30.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[24811]: osdmap e4: 0 total, 0 up, 0 in 2024-04-02T04:45:30.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[24811]: mgrmap e13: y(active, since 45s) 2024-04-02T04:45:30.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[24811]: overall HEALTH_OK 2024-04-02T04:45:30.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:30.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:45:30.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:30.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:45:30.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T04:45:30.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[29505]: mon.a calling monitor election 2024-04-02T04:45:30.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[29505]: mon.c calling monitor election 2024-04-02T04:45:30.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T04:45:30.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T04:45:30.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T04:45:30.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[29505]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:30.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T04:45:30.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[29505]: mon.b calling monitor election 2024-04-02T04:45:30.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T04:45:30.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[29505]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:30.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T04:45:30.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T04:45:30.337 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[29505]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-02T04:45:30.337 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[29505]: monmap e3: 3 mons at {a=[v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0],b=[v2:172.21.15.192:3300/0,v1:172.21.15.192:6789/0],c=[v2:172.21.15.86:3301/0,v1:172.21.15.86:6790/0]} 2024-04-02T04:45:30.337 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[29505]: fsmap 2024-04-02T04:45:30.337 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[29505]: osdmap e4: 0 total, 0 up, 0 in 2024-04-02T04:45:30.337 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[29505]: mgrmap e13: y(active, since 45s) 2024-04-02T04:45:30.337 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[29505]: overall HEALTH_OK 2024-04-02T04:45:30.337 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:30.337 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:45:30.337 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:30.337 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:29 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:45:30.684 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-04-02T04:45:30.684 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph config generate-minimal-conf 2024-04-02T04:45:30.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:30 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T04:45:30.984 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:30 smithi086 ceph-mon[24811]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:30.984 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:30 smithi086 ceph-mon[24811]: from='client.? 172.21.15.192:0/1625682340' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:45:30.984 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:30 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:30.984 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:30 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T04:45:30.984 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:30 smithi086 ceph-mon[29505]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:30.985 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:30 smithi086 ceph-mon[29505]: from='client.? 172.21.15.192:0/1625682340' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:45:30.985 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:30 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:30.985 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:30 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T04:45:32.194 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:32 smithi086 ceph-mon[24811]: Updating smithi086:/etc/ceph/ceph.conf 2024-04-02T04:45:32.194 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:32 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:32.195 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:32 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:32.195 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:32 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:32.195 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:32 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:32.195 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:32 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T04:45:32.195 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:32 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T04:45:32.195 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:32 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:32.195 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:32 smithi086 ceph-mon[29505]: Updating smithi086:/etc/ceph/ceph.conf 2024-04-02T04:45:32.195 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:32 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:32.195 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:32 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:32.195 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:32 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:32.195 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:32 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:32.195 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:32 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T04:45:32.196 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:32 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T04:45:32.196 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:32 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:32.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:32 smithi192 ceph-mon[25697]: Updating smithi086:/etc/ceph/ceph.conf 2024-04-02T04:45:32.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:32 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:32.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:32 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:32.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:32 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:32.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:32 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:32.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:32 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T04:45:32.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:32 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T04:45:32.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:32 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:32.781 INFO:teuthology.orchestra.run.smithi086.stdout:# minimal ceph.conf for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 2024-04-02T04:45:32.781 INFO:teuthology.orchestra.run.smithi086.stdout:[global] 2024-04-02T04:45:32.782 INFO:teuthology.orchestra.run.smithi086.stdout: fsid = 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 2024-04-02T04:45:32.782 INFO:teuthology.orchestra.run.smithi086.stdout: mon_host = [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] [v2:172.21.15.192:3300/0,v1:172.21.15.192:6789/0] [v2:172.21.15.86:3301/0,v1:172.21.15.86:6790/0] 2024-04-02T04:45:33.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:33 smithi086 ceph-mon[24811]: Updating smithi192:/etc/ceph/ceph.conf 2024-04-02T04:45:33.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:33 smithi086 ceph-mon[24811]: Reconfiguring mon.a (unknown last config time)... 2024-04-02T04:45:33.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:33 smithi086 ceph-mon[24811]: Reconfiguring daemon mon.a on smithi086 2024-04-02T04:45:33.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:33 smithi086 ceph-mon[24811]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:33.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:33 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1451287290' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:33.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:33 smithi086 ceph-mon[29505]: Updating smithi192:/etc/ceph/ceph.conf 2024-04-02T04:45:33.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:33 smithi086 ceph-mon[29505]: Reconfiguring mon.a (unknown last config time)... 2024-04-02T04:45:33.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:33 smithi086 ceph-mon[29505]: Reconfiguring daemon mon.a on smithi086 2024-04-02T04:45:33.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:33 smithi086 ceph-mon[29505]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:33.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:33 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1451287290' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:33.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:33 smithi192 ceph-mon[25697]: Updating smithi192:/etc/ceph/ceph.conf 2024-04-02T04:45:33.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:33 smithi192 ceph-mon[25697]: Reconfiguring mon.a (unknown last config time)... 2024-04-02T04:45:33.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:33 smithi192 ceph-mon[25697]: Reconfiguring daemon mon.a on smithi086 2024-04-02T04:45:33.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:33 smithi192 ceph-mon[25697]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:33.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:33 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1451287290' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:33.460 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-04-02T04:45:33.460 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-04-02T04:45:33.460 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd of=/etc/ceph/ceph.conf 2024-04-02T04:45:33.490 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-04-02T04:45:33.490 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-02T04:45:33.558 DEBUG:teuthology.orchestra.run.smithi192:> set -ex 2024-04-02T04:45:33.558 DEBUG:teuthology.orchestra.run.smithi192:> sudo dd of=/etc/ceph/ceph.conf 2024-04-02T04:45:33.591 DEBUG:teuthology.orchestra.run.smithi192:> set -ex 2024-04-02T04:45:33.591 DEBUG:teuthology.orchestra.run.smithi192:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-02T04:45:33.662 INFO:tasks.cephadm:Adding mgr.y on smithi086 2024-04-02T04:45:33.662 INFO:tasks.cephadm:Adding mgr.x on smithi192 2024-04-02T04:45:33.662 DEBUG:teuthology.orchestra.run.smithi192:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph orch apply mgr '2;smithi086=y;smithi192=x' 2024-04-02T04:45:35.154 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:34 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:35.154 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:34 smithi086 ceph-mon[24811]: Reconfiguring mon.c (monmap changed)... 2024-04-02T04:45:35.154 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:34 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T04:45:35.154 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:34 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T04:45:35.154 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:34 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:35.155 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:34 smithi086 ceph-mon[24811]: Reconfiguring daemon mon.c on smithi086 2024-04-02T04:45:35.155 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:34 smithi086 ceph-mon[24811]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:35.155 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:34 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:35.155 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:34 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:35.155 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:34 smithi086 ceph-mon[29505]: Reconfiguring mon.c (monmap changed)... 2024-04-02T04:45:35.155 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:34 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T04:45:35.155 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:34 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T04:45:35.155 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:34 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:35.155 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:34 smithi086 ceph-mon[29505]: Reconfiguring daemon mon.c on smithi086 2024-04-02T04:45:35.155 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:34 smithi086 ceph-mon[29505]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:35.155 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:34 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:35.164 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:34 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:35.164 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:34 smithi192 ceph-mon[25697]: Reconfiguring mon.c (monmap changed)... 2024-04-02T04:45:35.164 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:34 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T04:45:35.164 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:34 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T04:45:35.164 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:34 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:35.164 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:34 smithi192 ceph-mon[25697]: Reconfiguring daemon mon.c on smithi086 2024-04-02T04:45:35.164 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:34 smithi192 ceph-mon[25697]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:35.165 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:34 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:35.469 INFO:teuthology.orchestra.run.smithi192.stdout:Scheduled mgr update... 2024-04-02T04:45:36.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:36 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:36.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:36 smithi086 ceph-mon[24811]: Reconfiguring mon.b (monmap changed)... 2024-04-02T04:45:36.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:36 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T04:45:36.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:36 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T04:45:36.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:36 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:36.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:36 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:36.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:36 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:36.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:36 smithi086 ceph-mon[29505]: Reconfiguring mon.b (monmap changed)... 2024-04-02T04:45:36.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:36 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T04:45:36.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:36 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T04:45:36.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:36 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:36.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:36 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:36.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:36 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:36.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:36 smithi192 ceph-mon[25697]: Reconfiguring mon.b (monmap changed)... 2024-04-02T04:45:36.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:36 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T04:45:36.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:36 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T04:45:36.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:36 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:36.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:36 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:36.803 DEBUG:teuthology.orchestra.run.smithi192:mgr.x> sudo journalctl -f -n 0 -u ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mgr.x.service 2024-04-02T04:45:36.805 INFO:tasks.cephadm:Deploying OSDs... 2024-04-02T04:45:36.805 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-04-02T04:45:36.805 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/scratch_devs of=/dev/stdout 2024-04-02T04:45:36.824 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-02T04:45:36.824 DEBUG:teuthology.orchestra.run.smithi086:> stat /dev/vg_nvme/lv_1 2024-04-02T04:45:36.882 INFO:teuthology.orchestra.run.smithi086.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-02T04:45:36.883 INFO:teuthology.orchestra.run.smithi086.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T04:45:36.883 INFO:teuthology.orchestra.run.smithi086.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-04-02T04:45:36.883 INFO:teuthology.orchestra.run.smithi086.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T04:45:36.883 INFO:teuthology.orchestra.run.smithi086.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T04:45:36.883 INFO:teuthology.orchestra.run.smithi086.stdout:Access: 2024-04-02 04:45:13.973048842 +0000 2024-04-02T04:45:36.883 INFO:teuthology.orchestra.run.smithi086.stdout:Modify: 2024-04-02 04:42:58.324016290 +0000 2024-04-02T04:45:36.883 INFO:teuthology.orchestra.run.smithi086.stdout:Change: 2024-04-02 04:42:58.324016290 +0000 2024-04-02T04:45:36.883 INFO:teuthology.orchestra.run.smithi086.stdout: Birth: 2024-04-02 04:42:58.324016290 +0000 2024-04-02T04:45:36.883 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-02T04:45:36.954 INFO:teuthology.orchestra.run.smithi086.stderr:1+0 records in 2024-04-02T04:45:36.954 INFO:teuthology.orchestra.run.smithi086.stderr:1+0 records out 2024-04-02T04:45:36.954 INFO:teuthology.orchestra.run.smithi086.stderr:512 bytes copied, 0.000232006 s, 2.2 MB/s 2024-04-02T04:45:36.956 DEBUG:teuthology.orchestra.run.smithi086:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-02T04:45:37.015 DEBUG:teuthology.orchestra.run.smithi086:> stat /dev/vg_nvme/lv_2 2024-04-02T04:45:37.073 INFO:teuthology.orchestra.run.smithi086.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-02T04:45:37.073 INFO:teuthology.orchestra.run.smithi086.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T04:45:37.073 INFO:teuthology.orchestra.run.smithi086.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-04-02T04:45:37.073 INFO:teuthology.orchestra.run.smithi086.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T04:45:37.073 INFO:teuthology.orchestra.run.smithi086.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T04:45:37.073 INFO:teuthology.orchestra.run.smithi086.stdout:Access: 2024-04-02 04:45:13.974048821 +0000 2024-04-02T04:45:37.073 INFO:teuthology.orchestra.run.smithi086.stdout:Modify: 2024-04-02 04:42:58.685008539 +0000 2024-04-02T04:45:37.073 INFO:teuthology.orchestra.run.smithi086.stdout:Change: 2024-04-02 04:42:58.685008539 +0000 2024-04-02T04:45:37.073 INFO:teuthology.orchestra.run.smithi086.stdout: Birth: 2024-04-02 04:42:58.685008539 +0000 2024-04-02T04:45:37.074 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-02T04:45:37.141 INFO:teuthology.orchestra.run.smithi086.stderr:1+0 records in 2024-04-02T04:45:37.141 INFO:teuthology.orchestra.run.smithi086.stderr:1+0 records out 2024-04-02T04:45:37.141 INFO:teuthology.orchestra.run.smithi086.stderr:512 bytes copied, 0.000263136 s, 1.9 MB/s 2024-04-02T04:45:37.142 DEBUG:teuthology.orchestra.run.smithi086:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-02T04:45:37.199 DEBUG:teuthology.orchestra.run.smithi086:> stat /dev/vg_nvme/lv_3 2024-04-02T04:45:37.256 INFO:teuthology.orchestra.run.smithi086.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-02T04:45:37.256 INFO:teuthology.orchestra.run.smithi086.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T04:45:37.256 INFO:teuthology.orchestra.run.smithi086.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-04-02T04:45:37.256 INFO:teuthology.orchestra.run.smithi086.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T04:45:37.257 INFO:teuthology.orchestra.run.smithi086.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T04:45:37.257 INFO:teuthology.orchestra.run.smithi086.stdout:Access: 2024-04-02 04:45:13.974048821 +0000 2024-04-02T04:45:37.257 INFO:teuthology.orchestra.run.smithi086.stdout:Modify: 2024-04-02 04:42:59.011001540 +0000 2024-04-02T04:45:37.257 INFO:teuthology.orchestra.run.smithi086.stdout:Change: 2024-04-02 04:42:59.011001540 +0000 2024-04-02T04:45:37.257 INFO:teuthology.orchestra.run.smithi086.stdout: Birth: 2024-04-02 04:42:59.011001540 +0000 2024-04-02T04:45:37.257 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-02T04:45:37.322 INFO:teuthology.orchestra.run.smithi086.stderr:1+0 records in 2024-04-02T04:45:37.322 INFO:teuthology.orchestra.run.smithi086.stderr:1+0 records out 2024-04-02T04:45:37.322 INFO:teuthology.orchestra.run.smithi086.stderr:512 bytes copied, 0.000219969 s, 2.3 MB/s 2024-04-02T04:45:37.324 DEBUG:teuthology.orchestra.run.smithi086:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-02T04:45:37.380 DEBUG:teuthology.orchestra.run.smithi086:> stat /dev/vg_nvme/lv_4 2024-04-02T04:45:37.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:37 smithi192 ceph-mon[25697]: Reconfiguring daemon mon.b on smithi192 2024-04-02T04:45:37.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:37 smithi192 ceph-mon[25697]: from='client.34103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi086=y;smithi192=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:45:37.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:37 smithi192 ceph-mon[25697]: Saving service mgr spec with placement smithi086=y;smithi192=x;count:2 2024-04-02T04:45:37.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:37 smithi192 ceph-mon[25697]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:37.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:37 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:37.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:37 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:45:37.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:37 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:37.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:37 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:45:37.435 INFO:teuthology.orchestra.run.smithi086.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-02T04:45:37.435 INFO:teuthology.orchestra.run.smithi086.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T04:45:37.435 INFO:teuthology.orchestra.run.smithi086.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-04-02T04:45:37.435 INFO:teuthology.orchestra.run.smithi086.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T04:45:37.435 INFO:teuthology.orchestra.run.smithi086.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T04:45:37.435 INFO:teuthology.orchestra.run.smithi086.stdout:Access: 2024-04-02 04:45:13.975048799 +0000 2024-04-02T04:45:37.435 INFO:teuthology.orchestra.run.smithi086.stdout:Modify: 2024-04-02 04:42:59.340994454 +0000 2024-04-02T04:45:37.435 INFO:teuthology.orchestra.run.smithi086.stdout:Change: 2024-04-02 04:42:59.340994454 +0000 2024-04-02T04:45:37.435 INFO:teuthology.orchestra.run.smithi086.stdout: Birth: 2024-04-02 04:42:59.340994454 +0000 2024-04-02T04:45:37.435 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-02T04:45:37.498 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:37 smithi086 ceph-mon[24811]: Reconfiguring daemon mon.b on smithi192 2024-04-02T04:45:37.498 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:37 smithi086 ceph-mon[24811]: from='client.34103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi086=y;smithi192=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:45:37.498 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:37 smithi086 ceph-mon[24811]: Saving service mgr spec with placement smithi086=y;smithi192=x;count:2 2024-04-02T04:45:37.499 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:37 smithi086 ceph-mon[24811]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:37.499 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:37 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:37.499 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:37 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:45:37.499 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:37 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:37.499 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:37 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:45:37.499 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:37 smithi086 ceph-mon[29505]: Reconfiguring daemon mon.b on smithi192 2024-04-02T04:45:37.499 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:37 smithi086 ceph-mon[29505]: from='client.34103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi086=y;smithi192=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:45:37.499 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:37 smithi086 ceph-mon[29505]: Saving service mgr spec with placement smithi086=y;smithi192=x;count:2 2024-04-02T04:45:37.499 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:37 smithi086 ceph-mon[29505]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:37.499 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:37 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:37.499 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:37 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:45:37.500 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:37 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:37.500 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:37 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:45:37.500 INFO:teuthology.orchestra.run.smithi086.stderr:1+0 records in 2024-04-02T04:45:37.500 INFO:teuthology.orchestra.run.smithi086.stderr:1+0 records out 2024-04-02T04:45:37.500 INFO:teuthology.orchestra.run.smithi086.stderr:512 bytes copied, 0.000221707 s, 2.3 MB/s 2024-04-02T04:45:37.501 DEBUG:teuthology.orchestra.run.smithi086:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-02T04:45:37.557 DEBUG:teuthology.orchestra.run.smithi192:> set -ex 2024-04-02T04:45:37.557 DEBUG:teuthology.orchestra.run.smithi192:> dd if=/scratch_devs of=/dev/stdout 2024-04-02T04:45:37.572 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-02T04:45:37.572 DEBUG:teuthology.orchestra.run.smithi192:> stat /dev/vg_nvme/lv_1 2024-04-02T04:45:37.627 INFO:teuthology.orchestra.run.smithi192.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-02T04:45:37.627 INFO:teuthology.orchestra.run.smithi192.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T04:45:37.627 INFO:teuthology.orchestra.run.smithi192.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-04-02T04:45:37.627 INFO:teuthology.orchestra.run.smithi192.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T04:45:37.627 INFO:teuthology.orchestra.run.smithi192.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T04:45:37.627 INFO:teuthology.orchestra.run.smithi192.stdout:Access: 2024-04-02 04:45:24.272849897 +0000 2024-04-02T04:45:37.627 INFO:teuthology.orchestra.run.smithi192.stdout:Modify: 2024-04-02 04:42:58.332137090 +0000 2024-04-02T04:45:37.627 INFO:teuthology.orchestra.run.smithi192.stdout:Change: 2024-04-02 04:42:58.332137090 +0000 2024-04-02T04:45:37.627 INFO:teuthology.orchestra.run.smithi192.stdout: Birth: 2024-04-02 04:42:58.332137090 +0000 2024-04-02T04:45:37.628 DEBUG:teuthology.orchestra.run.smithi192:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-02T04:45:37.693 INFO:teuthology.orchestra.run.smithi192.stderr:1+0 records in 2024-04-02T04:45:37.693 INFO:teuthology.orchestra.run.smithi192.stderr:1+0 records out 2024-04-02T04:45:37.693 INFO:teuthology.orchestra.run.smithi192.stderr:512 bytes copied, 0.00018229 s, 2.8 MB/s 2024-04-02T04:45:37.694 DEBUG:teuthology.orchestra.run.smithi192:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-02T04:45:37.749 DEBUG:teuthology.orchestra.run.smithi192:> stat /dev/vg_nvme/lv_2 2024-04-02T04:45:37.805 INFO:teuthology.orchestra.run.smithi192.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-02T04:45:37.805 INFO:teuthology.orchestra.run.smithi192.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T04:45:37.805 INFO:teuthology.orchestra.run.smithi192.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-04-02T04:45:37.805 INFO:teuthology.orchestra.run.smithi192.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T04:45:37.805 INFO:teuthology.orchestra.run.smithi192.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T04:45:37.805 INFO:teuthology.orchestra.run.smithi192.stdout:Access: 2024-04-02 04:45:24.272849897 +0000 2024-04-02T04:45:37.806 INFO:teuthology.orchestra.run.smithi192.stdout:Modify: 2024-04-02 04:42:58.685130022 +0000 2024-04-02T04:45:37.806 INFO:teuthology.orchestra.run.smithi192.stdout:Change: 2024-04-02 04:42:58.685130022 +0000 2024-04-02T04:45:37.806 INFO:teuthology.orchestra.run.smithi192.stdout: Birth: 2024-04-02 04:42:58.685130022 +0000 2024-04-02T04:45:37.806 DEBUG:teuthology.orchestra.run.smithi192:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-02T04:45:37.871 INFO:teuthology.orchestra.run.smithi192.stderr:1+0 records in 2024-04-02T04:45:37.871 INFO:teuthology.orchestra.run.smithi192.stderr:1+0 records out 2024-04-02T04:45:37.871 INFO:teuthology.orchestra.run.smithi192.stderr:512 bytes copied, 0.000190589 s, 2.7 MB/s 2024-04-02T04:45:37.872 DEBUG:teuthology.orchestra.run.smithi192:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-02T04:45:37.927 DEBUG:teuthology.orchestra.run.smithi192:> stat /dev/vg_nvme/lv_3 2024-04-02T04:45:37.982 INFO:teuthology.orchestra.run.smithi192.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-02T04:45:37.982 INFO:teuthology.orchestra.run.smithi192.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T04:45:37.982 INFO:teuthology.orchestra.run.smithi192.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-04-02T04:45:37.983 INFO:teuthology.orchestra.run.smithi192.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T04:45:37.983 INFO:teuthology.orchestra.run.smithi192.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T04:45:37.983 INFO:teuthology.orchestra.run.smithi192.stdout:Access: 2024-04-02 04:45:24.273849876 +0000 2024-04-02T04:45:37.983 INFO:teuthology.orchestra.run.smithi192.stdout:Modify: 2024-04-02 04:42:59.015123415 +0000 2024-04-02T04:45:37.983 INFO:teuthology.orchestra.run.smithi192.stdout:Change: 2024-04-02 04:42:59.015123415 +0000 2024-04-02T04:45:37.983 INFO:teuthology.orchestra.run.smithi192.stdout: Birth: 2024-04-02 04:42:59.015123415 +0000 2024-04-02T04:45:37.983 DEBUG:teuthology.orchestra.run.smithi192:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-02T04:45:38.048 INFO:teuthology.orchestra.run.smithi192.stderr:1+0 records in 2024-04-02T04:45:38.048 INFO:teuthology.orchestra.run.smithi192.stderr:1+0 records out 2024-04-02T04:45:38.048 INFO:teuthology.orchestra.run.smithi192.stderr:512 bytes copied, 0.000248891 s, 2.1 MB/s 2024-04-02T04:45:38.050 DEBUG:teuthology.orchestra.run.smithi192:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-02T04:45:38.106 DEBUG:teuthology.orchestra.run.smithi192:> stat /dev/vg_nvme/lv_4 2024-04-02T04:45:38.160 INFO:teuthology.orchestra.run.smithi192.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-02T04:45:38.160 INFO:teuthology.orchestra.run.smithi192.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T04:45:38.161 INFO:teuthology.orchestra.run.smithi192.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-04-02T04:45:38.161 INFO:teuthology.orchestra.run.smithi192.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T04:45:38.161 INFO:teuthology.orchestra.run.smithi192.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T04:45:38.161 INFO:teuthology.orchestra.run.smithi192.stdout:Access: 2024-04-02 04:45:24.273849876 +0000 2024-04-02T04:45:38.161 INFO:teuthology.orchestra.run.smithi192.stdout:Modify: 2024-04-02 04:42:59.365116407 +0000 2024-04-02T04:45:38.161 INFO:teuthology.orchestra.run.smithi192.stdout:Change: 2024-04-02 04:42:59.365116407 +0000 2024-04-02T04:45:38.161 INFO:teuthology.orchestra.run.smithi192.stdout: Birth: 2024-04-02 04:42:59.365116407 +0000 2024-04-02T04:45:38.161 DEBUG:teuthology.orchestra.run.smithi192:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-02T04:45:38.226 INFO:teuthology.orchestra.run.smithi192.stderr:1+0 records in 2024-04-02T04:45:38.226 INFO:teuthology.orchestra.run.smithi192.stderr:1+0 records out 2024-04-02T04:45:38.226 INFO:teuthology.orchestra.run.smithi192.stderr:512 bytes copied, 0.000186355 s, 2.7 MB/s 2024-04-02T04:45:38.227 DEBUG:teuthology.orchestra.run.smithi192:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-02T04:45:38.283 INFO:tasks.cephadm:Deploying osd.0 on smithi086 with /dev/vg_nvme/lv_4... 2024-04-02T04:45:38.284 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-02T04:45:38.546 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:38 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:38.546 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:38 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T04:45:38.546 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:38 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-02T04:45:38.546 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:38 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T04:45:38.546 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:38 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:38.546 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:38 smithi192 ceph-mon[25697]: Deploying daemon mgr.x on smithi192 2024-04-02T04:45:38.547 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:38 smithi192 ceph-mon[25697]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:38.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:38 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:38.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:38 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T04:45:38.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:38 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-02T04:45:38.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:38 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T04:45:38.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:38 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:38.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:38 smithi086 ceph-mon[24811]: Deploying daemon mgr.x on smithi192 2024-04-02T04:45:38.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:38 smithi086 ceph-mon[24811]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:38.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:38 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:38.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:38 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T04:45:38.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:38 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-02T04:45:38.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:38 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T04:45:38.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:38 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:38.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:38 smithi086 ceph-mon[29505]: Deploying daemon mgr.x on smithi192 2024-04-02T04:45:38.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:38 smithi086 ceph-mon[29505]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:39.178 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:45:39 smithi192 podman[26867]: 2024-04-02 04:45:38.999669257 +0000 UTC m=+0.018124551 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-02T04:45:39.464 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:45:39 smithi192 podman[26867]: 2024-04-02 04:45:39.255932773 +0000 UTC m=+0.274388043 container init 2ebae22b00160c51becaf9945e219459ffdd09228a47cf925078b28b94209bd0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x, ceph=True, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, architecture=x86_64, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, vcs-type=git, distribution-scope=public, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, io.openshift.expose-services=, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-02T04:45:39.464 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:45:39 smithi192 podman[26867]: 2024-04-02 04:45:39.258878151 +0000 UTC m=+0.277333430 container start 2ebae22b00160c51becaf9945e219459ffdd09228a47cf925078b28b94209bd0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream) 2024-04-02T04:45:39.464 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:45:39 smithi192 ceph-mgr[26885]: -- 172.21.15.192:0/2618282572 <== mon.2 v2:172.21.15.192:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55b8ff6dc340 con 0x55b8ff6b4c00 2024-04-02T04:45:39.464 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:45:39 smithi192 bash[26867]: 2ebae22b00160c51becaf9945e219459ffdd09228a47cf925078b28b94209bd0 2024-04-02T04:45:39.464 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:45:39 smithi192 systemd[1]: Started Ceph mgr.x for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T04:45:40.152 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:45:39 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:45:39.769+0000 7f1d60c9f000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-02T04:45:40.153 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:45:39 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:45:39.863+0000 7f1d60c9f000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-02T04:45:40.171 INFO:teuthology.orchestra.run.smithi086.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-02T04:45:40.171 INFO:teuthology.orchestra.run.smithi086.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-02T04:45:40.171 INFO:teuthology.orchestra.run.smithi086.stderr: stderr: 10+0 records in 2024-04-02T04:45:40.171 INFO:teuthology.orchestra.run.smithi086.stderr:10+0 records out 2024-04-02T04:45:40.171 INFO:teuthology.orchestra.run.smithi086.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0131581 s, 797 MB/s 2024-04-02T04:45:40.172 INFO:teuthology.orchestra.run.smithi086.stderr:--> Zapping successful for: 2024-04-02T04:45:40.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:40 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:40.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:40 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:45:40.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:40 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:40.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:40 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:45:40.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:40 smithi192 ceph-mon[25697]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:40.679 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:45:40 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:45:40.479+0000 7f1d60c9f000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-02T04:45:40.679 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:45:40 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:45:40.568+0000 7f1d60c9f000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-02T04:45:40.679 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:45:40 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:45:40.651+0000 7f1d60c9f000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-02T04:45:40.716 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:40 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:40.716 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:40 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:45:40.716 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:40 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:40.716 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:40 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:45:40.716 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:40 smithi086 ceph-mon[24811]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:40.717 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:40 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:40.717 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:40 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:45:40.717 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:40 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:40.717 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:40 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:45:40.717 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:40 smithi086 ceph-mon[29505]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:40.740 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi086:vg_nvme/lv_4 2024-04-02T04:45:41.074 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:45:40 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:45:40.800+0000 7f1d60c9f000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-02T04:45:41.074 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:45:40 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:45:40.880+0000 7f1d60c9f000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-02T04:45:41.074 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:45:40 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:45:40.988+0000 7f1d60c9f000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-02T04:45:41.074 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:45:41 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:45:41.070+0000 7f1d60c9f000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-02T04:45:41.427 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:45:41 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:45:41.232+0000 7f1d60c9f000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-02T04:45:41.427 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:45:41 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:45:41.316+0000 7f1d60c9f000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-02T04:45:41.748 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:41 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:41.748 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:41 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:41.748 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:41 smithi086 ceph-mon[24811]: Reconfiguring mgr.y (unknown last config time)... 2024-04-02T04:45:41.748 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:41 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T04:45:41.748 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:41 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T04:45:41.748 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:41 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:41.748 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:41 smithi086 ceph-mon[24811]: Reconfiguring daemon mgr.y on smithi086 2024-04-02T04:45:41.749 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:41 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:41.749 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:41 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:41.749 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:41 smithi086 ceph-mon[29505]: Reconfiguring mgr.y (unknown last config time)... 2024-04-02T04:45:41.749 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:41 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T04:45:41.749 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:41 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T04:45:41.749 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:41 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:41.749 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:41 smithi086 ceph-mon[29505]: Reconfiguring daemon mgr.y on smithi086 2024-04-02T04:45:41.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:41 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:41.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:41 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:41.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:41 smithi192 ceph-mon[25697]: Reconfiguring mgr.y (unknown last config time)... 2024-04-02T04:45:41.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:41 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T04:45:41.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:41 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T04:45:41.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:41 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:41.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:41 smithi192 ceph-mon[25697]: Reconfiguring daemon mgr.y on smithi086 2024-04-02T04:45:42.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:42 smithi086 ceph-mon[24811]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:42.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:42 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:42.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:42 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:45:42.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:42 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:42.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:42 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:45:42.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:42 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:42.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:42 smithi086 ceph-mon[29505]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:42.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:42 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:42.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:42 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:45:42.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:42 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:42.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:42 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:45:42.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:42 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:42.666 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:45:42 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:45:42.342+0000 7f1d60c9f000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-02T04:45:42.666 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:45:42 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:45:42.473+0000 7f1d60c9f000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-02T04:45:42.666 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:45:42 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:45:42.662+0000 7f1d60c9f000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-02T04:45:42.666 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:42 smithi192 ceph-mon[25697]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:42.666 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:42 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:42.667 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:42 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:45:42.667 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:42 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:42.667 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:42 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:45:42.667 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:42 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:42.927 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:45:42 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:45:42.748+0000 7f1d60c9f000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-02T04:45:42.929 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:45:42 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:45:42.880+0000 7f1d60c9f000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-02T04:45:43.484 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:45:43 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:45:43.196+0000 7f1d60c9f000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-02T04:45:43.484 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:45:43 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:45:43.415+0000 7f1d60c9f000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-02T04:45:43.783 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:43 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T04:45:43.783 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:43 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T04:45:43.783 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:43 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:43.783 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:43 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T04:45:43.783 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:43 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T04:45:43.783 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:43 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:43.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:43 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T04:45:43.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:43 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T04:45:43.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:43 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:43.928 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:45:43 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:45:43.500+0000 7f1d60c9f000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-02T04:45:43.928 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:45:43 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:45:43.667+0000 7f1d60c9f000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-02T04:45:44.814 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:44 smithi086 ceph-mon[24811]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi086:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:45:44.815 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:44 smithi086 ceph-mon[24811]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:44.815 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:44 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:45:44.815 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:44 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:45:44.815 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:44 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:44.815 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:44 smithi086 ceph-mon[29505]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi086:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:45:44.815 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:44 smithi086 ceph-mon[29505]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:44.815 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:44 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:45:44.815 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:44 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:45:44.815 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:44 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:44.870 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:45:44 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:45:44.577+0000 7f1d60c9f000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-02T04:45:44.871 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:45:44 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:45:44.867+0000 7f1d60c9f000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-02T04:45:44.871 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:44 smithi192 ceph-mon[25697]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi086:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:45:44.871 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:44 smithi192 ceph-mon[25697]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:44.871 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:44 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:45:44.871 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:44 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:45:44.871 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:44 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:45.179 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:45:44 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:45:44.960+0000 7f1d60c9f000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-02T04:45:45.179 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:45:45 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:45:45.107+0000 7f1d60c9f000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-02T04:45:45.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:45 smithi086 ceph-mon[24811]: Standby manager daemon x started 2024-04-02T04:45:45.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:45 smithi086 ceph-mon[24811]: from='mgr.? 172.21.15.192:0/4037416041' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-02T04:45:45.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:45 smithi086 ceph-mon[24811]: from='mgr.? 172.21.15.192:0/4037416041' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T04:45:45.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:45 smithi086 ceph-mon[24811]: from='mgr.? 172.21.15.192:0/4037416041' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-02T04:45:45.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:45 smithi086 ceph-mon[24811]: from='mgr.? 172.21.15.192:0/4037416041' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T04:45:45.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:45 smithi086 ceph-mon[29505]: Standby manager daemon x started 2024-04-02T04:45:45.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:45 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.192:0/4037416041' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-02T04:45:45.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:45 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.192:0/4037416041' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T04:45:45.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:45 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.192:0/4037416041' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-02T04:45:45.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:45 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.192:0/4037416041' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T04:45:45.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:45 smithi192 ceph-mon[25697]: Standby manager daemon x started 2024-04-02T04:45:45.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:45 smithi192 ceph-mon[25697]: from='mgr.? 172.21.15.192:0/4037416041' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-02T04:45:45.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:45 smithi192 ceph-mon[25697]: from='mgr.? 172.21.15.192:0/4037416041' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T04:45:45.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:45 smithi192 ceph-mon[25697]: from='mgr.? 172.21.15.192:0/4037416041' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-02T04:45:45.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:45 smithi192 ceph-mon[25697]: from='mgr.? 172.21.15.192:0/4037416041' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T04:45:46.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:46 smithi086 ceph-mon[24811]: mgrmap e14: y(active, since 61s), standbys: x 2024-04-02T04:45:46.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:46 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T04:45:46.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:46 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1318619261' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e6ee7c83-68a3-4d34-8cdc-21972cf722bf"}]: dispatch 2024-04-02T04:45:46.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:46 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1318619261' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e6ee7c83-68a3-4d34-8cdc-21972cf722bf"}]': finished 2024-04-02T04:45:46.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:46 smithi086 ceph-mon[24811]: osdmap e5: 1 total, 0 up, 1 in 2024-04-02T04:45:46.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:46 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T04:45:46.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:46 smithi086 ceph-mon[24811]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:46.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:46 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/3387526255' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T04:45:46.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:46 smithi086 ceph-mon[29505]: mgrmap e14: y(active, since 61s), standbys: x 2024-04-02T04:45:46.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:46 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T04:45:46.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:46 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1318619261' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e6ee7c83-68a3-4d34-8cdc-21972cf722bf"}]: dispatch 2024-04-02T04:45:46.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:46 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1318619261' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e6ee7c83-68a3-4d34-8cdc-21972cf722bf"}]': finished 2024-04-02T04:45:46.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:46 smithi086 ceph-mon[29505]: osdmap e5: 1 total, 0 up, 1 in 2024-04-02T04:45:46.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:46 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T04:45:46.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:46 smithi086 ceph-mon[29505]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:46.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:46 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/3387526255' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T04:45:46.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:46 smithi192 ceph-mon[25697]: mgrmap e14: y(active, since 61s), standbys: x 2024-04-02T04:45:46.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:46 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T04:45:46.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:46 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1318619261' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e6ee7c83-68a3-4d34-8cdc-21972cf722bf"}]: dispatch 2024-04-02T04:45:46.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:46 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1318619261' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e6ee7c83-68a3-4d34-8cdc-21972cf722bf"}]': finished 2024-04-02T04:45:46.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:46 smithi192 ceph-mon[25697]: osdmap e5: 1 total, 0 up, 1 in 2024-04-02T04:45:46.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:46 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T04:45:46.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:46 smithi192 ceph-mon[25697]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:46.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:46 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/3387526255' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T04:45:49.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:49 smithi086 ceph-mon[24811]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:49.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:49 smithi086 ceph-mon[29505]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:49.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:49 smithi192 ceph-mon[25697]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:51.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:51 smithi086 ceph-mon[24811]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:51.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:51 smithi086 ceph-mon[29505]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:51.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:51 smithi192 ceph-mon[25697]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:52.379 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:52 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-02T04:45:52.379 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:52 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:52.379 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:52 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-02T04:45:52.379 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:52 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:52.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:52 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-02T04:45:52.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:52 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:53.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:53 smithi086 ceph-mon[24811]: Deploying daemon osd.0 on smithi086 2024-04-02T04:45:53.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:53 smithi086 ceph-mon[24811]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:53.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:53 smithi086 ceph-mon[29505]: Deploying daemon osd.0 on smithi086 2024-04-02T04:45:53.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:53 smithi086 ceph-mon[29505]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:53.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:53 smithi192 ceph-mon[25697]: Deploying daemon osd.0 on smithi086 2024-04-02T04:45:53.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:53 smithi192 ceph-mon[25697]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:55.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:55 smithi192 ceph-mon[25697]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:55.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:55 smithi086 ceph-mon[24811]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:55.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:55 smithi086 ceph-mon[29505]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:56.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:56 smithi086 ceph-mon[24811]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:56.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:56 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:56.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:56 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:45:56.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:56 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:56.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:56 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:45:56.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:56 smithi086 ceph-mon[29505]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:56.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:56 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:56.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:56 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:45:56.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:56 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:56.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:56 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:45:56.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:56 smithi192 ceph-mon[25697]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:56.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:56 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:45:56.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:56 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:45:56.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:56 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:45:56.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:56 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:45:59.321 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:45:59 smithi086 ceph-mon[24811]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:59.322 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:45:59 smithi086 ceph-mon[29505]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:59.329 INFO:teuthology.orchestra.run.smithi086.stdout:Created osd(s) 0 on host 'smithi086' 2024-04-02T04:45:59.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:45:59 smithi192 ceph-mon[25697]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:45:59.999 DEBUG:teuthology.orchestra.run.smithi086:osd.0> sudo journalctl -f -n 0 -u ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.0.service 2024-04-02T04:46:00.002 INFO:tasks.cephadm:Deploying osd.1 on smithi086 with /dev/vg_nvme/lv_3... 2024-04-02T04:46:00.002 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-02T04:46:00.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:00 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:00.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:00 smithi086 ceph-mon[24811]: from='osd.0 [v2:172.21.15.86:6802/2621947859,v1:172.21.15.86:6803/2621947859]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-02T04:46:00.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:00 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:00.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:00 smithi086 ceph-mon[24811]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:46:00.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:00 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:00.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:00 smithi086 ceph-mon[29505]: from='osd.0 [v2:172.21.15.86:6802/2621947859,v1:172.21.15.86:6803/2621947859]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-02T04:46:00.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:00 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:00.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:00 smithi086 ceph-mon[29505]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:46:00.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:00 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:00.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:00 smithi192 ceph-mon[25697]: from='osd.0 [v2:172.21.15.86:6802/2621947859,v1:172.21.15.86:6803/2621947859]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-02T04:46:00.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:00 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:00.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:00 smithi192 ceph-mon[25697]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:46:01.537 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:01 smithi086 ceph-mon[24811]: from='osd.0 [v2:172.21.15.86:6802/2621947859,v1:172.21.15.86:6803/2621947859]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-02T04:46:01.537 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:01 smithi086 ceph-mon[24811]: osdmap e6: 1 total, 0 up, 1 in 2024-04-02T04:46:01.537 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:01 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T04:46:01.537 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:01 smithi086 ceph-mon[24811]: from='osd.0 [v2:172.21.15.86:6802/2621947859,v1:172.21.15.86:6803/2621947859]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T04:46:01.538 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:01 smithi086 ceph-mon[29505]: from='osd.0 [v2:172.21.15.86:6802/2621947859,v1:172.21.15.86:6803/2621947859]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-02T04:46:01.538 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:01 smithi086 ceph-mon[29505]: osdmap e6: 1 total, 0 up, 1 in 2024-04-02T04:46:01.538 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:01 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T04:46:01.538 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:01 smithi086 ceph-mon[29505]: from='osd.0 [v2:172.21.15.86:6802/2621947859,v1:172.21.15.86:6803/2621947859]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T04:46:01.538 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 04:46:01 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0[32375]: 2024-04-02T04:46:01.331+0000 7efd71e5b700 -1 osd.0 0 waiting for initial osdmap 2024-04-02T04:46:01.538 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 04:46:01 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0[32375]: 2024-04-02T04:46:01.335+0000 7efd6a7ee700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T04:46:01.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:01 smithi192 ceph-mon[25697]: from='osd.0 [v2:172.21.15.86:6802/2621947859,v1:172.21.15.86:6803/2621947859]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-02T04:46:01.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:01 smithi192 ceph-mon[25697]: osdmap e6: 1 total, 0 up, 1 in 2024-04-02T04:46:01.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:01 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T04:46:01.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:01 smithi192 ceph-mon[25697]: from='osd.0 [v2:172.21.15.86:6802/2621947859,v1:172.21.15.86:6803/2621947859]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T04:46:01.913 INFO:teuthology.orchestra.run.smithi086.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-02T04:46:01.913 INFO:teuthology.orchestra.run.smithi086.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-02T04:46:01.913 INFO:teuthology.orchestra.run.smithi086.stderr: stderr: 10+0 records in 2024-04-02T04:46:01.913 INFO:teuthology.orchestra.run.smithi086.stderr:10+0 records out 2024-04-02T04:46:01.913 INFO:teuthology.orchestra.run.smithi086.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0123363 s, 850 MB/s 2024-04-02T04:46:01.913 INFO:teuthology.orchestra.run.smithi086.stderr:--> Zapping successful for: 2024-04-02T04:46:02.560 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:02 smithi086 ceph-mon[29505]: from='osd.0 [v2:172.21.15.86:6802/2621947859,v1:172.21.15.86:6803/2621947859]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]': finished 2024-04-02T04:46:02.560 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:02 smithi086 ceph-mon[29505]: osdmap e7: 1 total, 0 up, 1 in 2024-04-02T04:46:02.561 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:02 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T04:46:02.561 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:02 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T04:46:02.561 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:02 smithi086 ceph-mon[29505]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:46:02.561 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:02 smithi086 ceph-mon[24811]: from='osd.0 [v2:172.21.15.86:6802/2621947859,v1:172.21.15.86:6803/2621947859]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]': finished 2024-04-02T04:46:02.561 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:02 smithi086 ceph-mon[24811]: osdmap e7: 1 total, 0 up, 1 in 2024-04-02T04:46:02.561 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:02 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T04:46:02.561 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:02 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T04:46:02.561 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:02 smithi086 ceph-mon[24811]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:46:02.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:02 smithi192 ceph-mon[25697]: from='osd.0 [v2:172.21.15.86:6802/2621947859,v1:172.21.15.86:6803/2621947859]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]': finished 2024-04-02T04:46:02.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:02 smithi192 ceph-mon[25697]: osdmap e7: 1 total, 0 up, 1 in 2024-04-02T04:46:02.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:02 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T04:46:02.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:02 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T04:46:02.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:02 smithi192 ceph-mon[25697]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:46:02.834 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi086:vg_nvme/lv_3 2024-04-02T04:46:03.465 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:03 smithi086 ceph-mon[29505]: purged_snaps scrub starts 2024-04-02T04:46:03.465 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:03 smithi086 ceph-mon[29505]: purged_snaps scrub ok 2024-04-02T04:46:03.466 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:03 smithi086 ceph-mon[29505]: osd.0 [v2:172.21.15.86:6802/2621947859,v1:172.21.15.86:6803/2621947859] boot 2024-04-02T04:46:03.466 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:03 smithi086 ceph-mon[29505]: osdmap e8: 1 total, 1 up, 1 in 2024-04-02T04:46:03.466 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:03 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T04:46:03.466 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:03 smithi086 ceph-mon[24811]: purged_snaps scrub starts 2024-04-02T04:46:03.466 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:03 smithi086 ceph-mon[24811]: purged_snaps scrub ok 2024-04-02T04:46:03.466 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:03 smithi086 ceph-mon[24811]: osd.0 [v2:172.21.15.86:6802/2621947859,v1:172.21.15.86:6803/2621947859] boot 2024-04-02T04:46:03.466 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:03 smithi086 ceph-mon[24811]: osdmap e8: 1 total, 1 up, 1 in 2024-04-02T04:46:03.466 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:03 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T04:46:03.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:03 smithi192 ceph-mon[25697]: purged_snaps scrub starts 2024-04-02T04:46:03.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:03 smithi192 ceph-mon[25697]: purged_snaps scrub ok 2024-04-02T04:46:03.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:03 smithi192 ceph-mon[25697]: osd.0 [v2:172.21.15.86:6802/2621947859,v1:172.21.15.86:6803/2621947859] boot 2024-04-02T04:46:03.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:03 smithi192 ceph-mon[25697]: osdmap e8: 1 total, 1 up, 1 in 2024-04-02T04:46:03.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:03 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T04:46:04.631 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:04 smithi086 ceph-mon[24811]: osdmap e9: 1 total, 1 up, 1 in 2024-04-02T04:46:04.631 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:04 smithi086 ceph-mon[24811]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:46:04.631 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:04 smithi086 ceph-mon[29505]: osdmap e9: 1 total, 1 up, 1 in 2024-04-02T04:46:04.632 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:04 smithi086 ceph-mon[29505]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:46:04.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:04 smithi192 ceph-mon[25697]: osdmap e9: 1 total, 1 up, 1 in 2024-04-02T04:46:04.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:04 smithi192 ceph-mon[25697]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:46:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:05 smithi192 ceph-mon[25697]: Detected new or changed devices on smithi086 2024-04-02T04:46:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:05 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:05 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:46:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:05 smithi192 ceph-mon[25697]: Adjusting osd_memory_target on smithi086 to 16064M 2024-04-02T04:46:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:05 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:05 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:05 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:46:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:05 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:46:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:05 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:46:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:05 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:05 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T04:46:06.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:05 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T04:46:06.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:05 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:46:06.192 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:05 smithi086 ceph-mon[24811]: Detected new or changed devices on smithi086 2024-04-02T04:46:06.192 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:05 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:06.192 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:05 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:46:06.192 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:05 smithi086 ceph-mon[24811]: Adjusting osd_memory_target on smithi086 to 16064M 2024-04-02T04:46:06.192 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:05 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:06.192 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:05 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:06.192 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:05 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:46:06.192 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:05 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:46:06.192 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:05 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:46:06.192 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:05 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:06.192 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:05 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T04:46:06.192 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:05 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T04:46:06.193 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:05 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:46:06.193 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:05 smithi086 ceph-mon[29505]: Detected new or changed devices on smithi086 2024-04-02T04:46:06.193 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:05 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:06.193 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:05 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:46:06.193 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:05 smithi086 ceph-mon[29505]: Adjusting osd_memory_target on smithi086 to 16064M 2024-04-02T04:46:06.193 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:05 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:06.193 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:05 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:06.193 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:05 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:46:06.193 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:05 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:46:06.193 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:05 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:46:06.194 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:05 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:06.194 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:05 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T04:46:06.194 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:05 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T04:46:06.194 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:05 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:46:07.135 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:06 smithi086 ceph-mon[24811]: from='client.24148 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi086:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:46:07.135 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:06 smithi086 ceph-mon[24811]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:46:07.135 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:06 smithi086 ceph-mon[29505]: from='client.24148 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi086:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:46:07.135 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:06 smithi086 ceph-mon[29505]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:46:07.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:06 smithi192 ceph-mon[25697]: from='client.24148 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi086:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:46:07.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:06 smithi192 ceph-mon[25697]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:46:08.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:08 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/3746782706' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fb117c77-5aff-44fc-bcfd-23805e94220d"}]: dispatch 2024-04-02T04:46:08.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:08 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/3746782706' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fb117c77-5aff-44fc-bcfd-23805e94220d"}]': finished 2024-04-02T04:46:08.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:08 smithi086 ceph-mon[29505]: osdmap e10: 2 total, 1 up, 2 in 2024-04-02T04:46:08.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:08 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T04:46:08.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:08 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/3746782706' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fb117c77-5aff-44fc-bcfd-23805e94220d"}]: dispatch 2024-04-02T04:46:08.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:08 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/3746782706' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fb117c77-5aff-44fc-bcfd-23805e94220d"}]': finished 2024-04-02T04:46:08.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:08 smithi086 ceph-mon[24811]: osdmap e10: 2 total, 1 up, 2 in 2024-04-02T04:46:08.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:08 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T04:46:08.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:08 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/3746782706' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fb117c77-5aff-44fc-bcfd-23805e94220d"}]: dispatch 2024-04-02T04:46:08.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:08 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/3746782706' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fb117c77-5aff-44fc-bcfd-23805e94220d"}]': finished 2024-04-02T04:46:08.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:08 smithi192 ceph-mon[25697]: osdmap e10: 2 total, 1 up, 2 in 2024-04-02T04:46:08.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:08 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T04:46:09.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:09 smithi192 ceph-mon[25697]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:46:09.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:09 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/2594861590' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T04:46:09.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:09 smithi086 ceph-mon[24811]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:46:09.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:09 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/2594861590' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T04:46:09.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:09 smithi086 ceph-mon[29505]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:46:09.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:09 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/2594861590' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T04:46:11.326 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:11 smithi086 ceph-mon[24811]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:46:11.327 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:11 smithi086 ceph-mon[29505]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:46:11.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:11 smithi192 ceph-mon[25697]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:46:13.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:13 smithi086 ceph-mon[24811]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:46:13.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:13 smithi086 ceph-mon[29505]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:46:13.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:13 smithi192 ceph-mon[25697]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:46:14.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-02T04:46:14.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:46:14.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:14 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-02T04:46:14.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:14 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:46:14.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:14 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-02T04:46:14.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:14 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:46:15.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:15 smithi192 ceph-mon[25697]: Deploying daemon osd.1 on smithi086 2024-04-02T04:46:15.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:15 smithi192 ceph-mon[25697]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:46:15.443 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:15 smithi086 ceph-mon[24811]: Deploying daemon osd.1 on smithi086 2024-04-02T04:46:15.443 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:15 smithi086 ceph-mon[24811]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:46:15.443 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:15 smithi086 ceph-mon[29505]: Deploying daemon osd.1 on smithi086 2024-04-02T04:46:15.443 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:15 smithi086 ceph-mon[29505]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:46:16.831 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:16 smithi086 ceph-mon[29505]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:46:16.832 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:16 smithi086 ceph-mon[24811]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:46:16.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:16 smithi192 ceph-mon[25697]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:46:19.255 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:19 smithi086 ceph-mon[29505]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:46:19.255 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:19 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:19.255 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:19 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:46:19.255 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:19 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:46:19.255 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:19 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:46:19.255 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:19 smithi086 ceph-mon[24811]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:46:19.255 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:19 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:19.255 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:19 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:46:19.255 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:19 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:46:19.255 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:19 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:46:19.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:19 smithi192 ceph-mon[25697]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:46:19.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:19 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:19.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:19 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:46:19.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:19 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:46:19.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:19 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:46:21.284 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:21 smithi086 ceph-mon[24811]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:46:21.284 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:21 smithi086 ceph-mon[29505]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:46:21.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:21 smithi192 ceph-mon[25697]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:46:21.892 INFO:teuthology.orchestra.run.smithi086.stdout:Created osd(s) 1 on host 'smithi086' 2024-04-02T04:46:22.515 DEBUG:teuthology.orchestra.run.smithi086:osd.1> sudo journalctl -f -n 0 -u ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.1.service 2024-04-02T04:46:22.517 INFO:tasks.cephadm:Deploying osd.2 on smithi086 with /dev/vg_nvme/lv_2... 2024-04-02T04:46:22.517 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-02T04:46:23.168 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:22 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:23.168 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:22 smithi086 ceph-mon[24811]: from='osd.1 [v2:172.21.15.86:6810/3495515130,v1:172.21.15.86:6811/3495515130]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-02T04:46:23.168 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:22 smithi086 ceph-mon[24811]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:46:23.168 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:22 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:23.168 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:22 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:23.168 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:22 smithi086 ceph-mon[29505]: from='osd.1 [v2:172.21.15.86:6810/3495515130,v1:172.21.15.86:6811/3495515130]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-02T04:46:23.168 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:22 smithi086 ceph-mon[29505]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:46:23.168 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:22 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:23.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:22 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:23.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:22 smithi192 ceph-mon[25697]: from='osd.1 [v2:172.21.15.86:6810/3495515130,v1:172.21.15.86:6811/3495515130]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-02T04:46:23.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:22 smithi192 ceph-mon[25697]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:46:23.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:22 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:24.027 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:23 smithi086 ceph-mon[24811]: from='osd.1 [v2:172.21.15.86:6810/3495515130,v1:172.21.15.86:6811/3495515130]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-02T04:46:24.027 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:23 smithi086 ceph-mon[24811]: osdmap e11: 2 total, 1 up, 2 in 2024-04-02T04:46:24.027 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:23 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T04:46:24.027 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:23 smithi086 ceph-mon[24811]: from='osd.1 [v2:172.21.15.86:6810/3495515130,v1:172.21.15.86:6811/3495515130]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T04:46:24.027 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:23 smithi086 ceph-mon[29505]: from='osd.1 [v2:172.21.15.86:6810/3495515130,v1:172.21.15.86:6811/3495515130]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-02T04:46:24.027 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:23 smithi086 ceph-mon[29505]: osdmap e11: 2 total, 1 up, 2 in 2024-04-02T04:46:24.027 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:23 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T04:46:24.028 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:23 smithi086 ceph-mon[29505]: from='osd.1 [v2:172.21.15.86:6810/3495515130,v1:172.21.15.86:6811/3495515130]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T04:46:24.028 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 04:46:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1[35345]: 2024-04-02T04:46:23.903+0000 7f55e27b9700 -1 osd.1 0 waiting for initial osdmap 2024-04-02T04:46:24.028 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 04:46:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1[35345]: 2024-04-02T04:46:23.907+0000 7f55dd951700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T04:46:24.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:23 smithi192 ceph-mon[25697]: from='osd.1 [v2:172.21.15.86:6810/3495515130,v1:172.21.15.86:6811/3495515130]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-02T04:46:24.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:23 smithi192 ceph-mon[25697]: osdmap e11: 2 total, 1 up, 2 in 2024-04-02T04:46:24.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:23 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T04:46:24.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:23 smithi192 ceph-mon[25697]: from='osd.1 [v2:172.21.15.86:6810/3495515130,v1:172.21.15.86:6811/3495515130]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T04:46:24.538 INFO:teuthology.orchestra.run.smithi086.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-02T04:46:24.538 INFO:teuthology.orchestra.run.smithi086.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-02T04:46:24.538 INFO:teuthology.orchestra.run.smithi086.stderr: stderr: 10+0 records in 2024-04-02T04:46:24.538 INFO:teuthology.orchestra.run.smithi086.stderr:10+0 records out 2024-04-02T04:46:24.538 INFO:teuthology.orchestra.run.smithi086.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0124099 s, 845 MB/s 2024-04-02T04:46:24.538 INFO:teuthology.orchestra.run.smithi086.stderr:--> Zapping successful for: 2024-04-02T04:46:25.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:24 smithi192 ceph-mon[25697]: from='osd.1 [v2:172.21.15.86:6810/3495515130,v1:172.21.15.86:6811/3495515130]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]': finished 2024-04-02T04:46:25.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:24 smithi192 ceph-mon[25697]: osdmap e12: 2 total, 1 up, 2 in 2024-04-02T04:46:25.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:24 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T04:46:25.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:24 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T04:46:25.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:24 smithi192 ceph-mon[25697]: pgmap v52: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:46:25.223 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:24 smithi086 ceph-mon[24811]: from='osd.1 [v2:172.21.15.86:6810/3495515130,v1:172.21.15.86:6811/3495515130]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]': finished 2024-04-02T04:46:25.223 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:24 smithi086 ceph-mon[24811]: osdmap e12: 2 total, 1 up, 2 in 2024-04-02T04:46:25.223 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:24 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T04:46:25.223 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:24 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T04:46:25.223 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:24 smithi086 ceph-mon[24811]: pgmap v52: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:46:25.224 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:24 smithi086 ceph-mon[29505]: from='osd.1 [v2:172.21.15.86:6810/3495515130,v1:172.21.15.86:6811/3495515130]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]': finished 2024-04-02T04:46:25.224 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:24 smithi086 ceph-mon[29505]: osdmap e12: 2 total, 1 up, 2 in 2024-04-02T04:46:25.224 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:24 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T04:46:25.224 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:24 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T04:46:25.224 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:24 smithi086 ceph-mon[29505]: pgmap v52: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:46:26.045 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:25 smithi086 ceph-mon[29505]: purged_snaps scrub starts 2024-04-02T04:46:26.045 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:25 smithi086 ceph-mon[29505]: purged_snaps scrub ok 2024-04-02T04:46:26.045 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:25 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T04:46:26.045 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:25 smithi086 ceph-mon[29505]: osd.1 [v2:172.21.15.86:6810/3495515130,v1:172.21.15.86:6811/3495515130] boot 2024-04-02T04:46:26.045 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:25 smithi086 ceph-mon[29505]: osdmap e13: 2 total, 2 up, 2 in 2024-04-02T04:46:26.045 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:25 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T04:46:26.045 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:25 smithi086 ceph-mon[24811]: purged_snaps scrub starts 2024-04-02T04:46:26.046 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:25 smithi086 ceph-mon[24811]: purged_snaps scrub ok 2024-04-02T04:46:26.046 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:25 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T04:46:26.046 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:25 smithi086 ceph-mon[24811]: osd.1 [v2:172.21.15.86:6810/3495515130,v1:172.21.15.86:6811/3495515130] boot 2024-04-02T04:46:26.046 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:25 smithi086 ceph-mon[24811]: osdmap e13: 2 total, 2 up, 2 in 2024-04-02T04:46:26.046 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:25 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T04:46:26.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:25 smithi192 ceph-mon[25697]: purged_snaps scrub starts 2024-04-02T04:46:26.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:25 smithi192 ceph-mon[25697]: purged_snaps scrub ok 2024-04-02T04:46:26.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:25 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T04:46:26.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:25 smithi192 ceph-mon[25697]: osd.1 [v2:172.21.15.86:6810/3495515130,v1:172.21.15.86:6811/3495515130] boot 2024-04-02T04:46:26.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:25 smithi192 ceph-mon[25697]: osdmap e13: 2 total, 2 up, 2 in 2024-04-02T04:46:26.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:25 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T04:46:26.352 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi086:vg_nvme/lv_2 2024-04-02T04:46:27.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:26 smithi192 ceph-mon[25697]: pgmap v54: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-02T04:46:27.208 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:26 smithi086 ceph-mon[24811]: pgmap v54: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-02T04:46:27.208 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:26 smithi086 ceph-mon[29505]: pgmap v54: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-02T04:46:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:27 smithi192 ceph-mon[25697]: osdmap e14: 2 total, 2 up, 2 in 2024-04-02T04:46:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:27 smithi192 ceph-mon[25697]: Detected new or changed devices on smithi086 2024-04-02T04:46:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:27 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:27 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:46:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:27 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:46:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:27 smithi192 ceph-mon[25697]: Adjusting osd_memory_target on smithi086 to 8032M 2024-04-02T04:46:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:27 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:27 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:27 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:46:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:27 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:46:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:27 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:46:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:27 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:28.212 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:27 smithi086 ceph-mon[29505]: osdmap e14: 2 total, 2 up, 2 in 2024-04-02T04:46:28.212 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:27 smithi086 ceph-mon[29505]: Detected new or changed devices on smithi086 2024-04-02T04:46:28.212 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:27 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:28.212 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:27 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:46:28.212 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:27 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:46:28.212 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:27 smithi086 ceph-mon[29505]: Adjusting osd_memory_target on smithi086 to 8032M 2024-04-02T04:46:28.212 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:27 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:28.212 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:27 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:28.212 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:27 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:46:28.212 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:27 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:46:28.213 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:27 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:46:28.213 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:27 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:28.213 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:27 smithi086 ceph-mon[24811]: osdmap e14: 2 total, 2 up, 2 in 2024-04-02T04:46:28.213 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:27 smithi086 ceph-mon[24811]: Detected new or changed devices on smithi086 2024-04-02T04:46:28.213 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:27 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:28.213 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:27 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:46:28.213 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:27 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:46:28.213 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:27 smithi086 ceph-mon[24811]: Adjusting osd_memory_target on smithi086 to 8032M 2024-04-02T04:46:28.213 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:27 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:28.213 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:27 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:28.213 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:27 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:46:28.213 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:27 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:46:28.214 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:27 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:46:28.214 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:27 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:29.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:28 smithi086 ceph-mon[24811]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-02T04:46:29.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:28 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T04:46:29.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:28 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T04:46:29.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:28 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:46:29.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:28 smithi086 ceph-mon[29505]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-02T04:46:29.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:28 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T04:46:29.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:28 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T04:46:29.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:28 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:46:29.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:28 smithi192 ceph-mon[25697]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-02T04:46:29.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:28 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T04:46:29.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:28 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T04:46:29.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:28 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:46:30.191 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:29 smithi086 ceph-mon[29505]: from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi086:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:46:30.191 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:29 smithi086 ceph-mon[24811]: from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi086:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:46:30.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:29 smithi192 ceph-mon[25697]: from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi086:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:46:31.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:30 smithi086 ceph-mon[24811]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T04:46:31.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:30 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/2615488272' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a0ee7998-0bb3-4b9d-b536-d305b6679aa6"}]: dispatch 2024-04-02T04:46:31.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:30 smithi086 ceph-mon[24811]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a0ee7998-0bb3-4b9d-b536-d305b6679aa6"}]: dispatch 2024-04-02T04:46:31.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:30 smithi086 ceph-mon[24811]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a0ee7998-0bb3-4b9d-b536-d305b6679aa6"}]': finished 2024-04-02T04:46:31.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:30 smithi086 ceph-mon[24811]: osdmap e15: 3 total, 2 up, 3 in 2024-04-02T04:46:31.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:30 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T04:46:31.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:30 smithi086 ceph-mon[29505]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T04:46:31.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:30 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/2615488272' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a0ee7998-0bb3-4b9d-b536-d305b6679aa6"}]: dispatch 2024-04-02T04:46:31.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:30 smithi086 ceph-mon[29505]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a0ee7998-0bb3-4b9d-b536-d305b6679aa6"}]: dispatch 2024-04-02T04:46:31.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:30 smithi086 ceph-mon[29505]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a0ee7998-0bb3-4b9d-b536-d305b6679aa6"}]': finished 2024-04-02T04:46:31.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:30 smithi086 ceph-mon[29505]: osdmap e15: 3 total, 2 up, 3 in 2024-04-02T04:46:31.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:30 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T04:46:31.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:30 smithi192 ceph-mon[25697]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T04:46:31.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:30 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/2615488272' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a0ee7998-0bb3-4b9d-b536-d305b6679aa6"}]: dispatch 2024-04-02T04:46:31.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:30 smithi192 ceph-mon[25697]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a0ee7998-0bb3-4b9d-b536-d305b6679aa6"}]: dispatch 2024-04-02T04:46:31.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:30 smithi192 ceph-mon[25697]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a0ee7998-0bb3-4b9d-b536-d305b6679aa6"}]': finished 2024-04-02T04:46:31.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:30 smithi192 ceph-mon[25697]: osdmap e15: 3 total, 2 up, 3 in 2024-04-02T04:46:31.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:30 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T04:46:32.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:31 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/596980228' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T04:46:32.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:31 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/596980228' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T04:46:32.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:31 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/596980228' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T04:46:33.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:32 smithi086 ceph-mon[24811]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T04:46:33.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:32 smithi086 ceph-mon[29505]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T04:46:33.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:32 smithi192 ceph-mon[25697]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T04:46:35.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:35 smithi192 ceph-mon[25697]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T04:46:35.530 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:35 smithi086 ceph-mon[24811]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T04:46:35.531 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:35 smithi086 ceph-mon[29505]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T04:46:36.879 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:36 smithi086 ceph-mon[29505]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T04:46:36.879 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:36 smithi086 ceph-mon[24811]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T04:46:36.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:36 smithi192 ceph-mon[25697]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T04:46:37.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:37 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-02T04:46:37.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:37 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:46:37.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:37 smithi086 ceph-mon[24811]: Deploying daemon osd.2 on smithi086 2024-04-02T04:46:37.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:37 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-02T04:46:37.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:37 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:46:37.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:37 smithi086 ceph-mon[29505]: Deploying daemon osd.2 on smithi086 2024-04-02T04:46:37.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:37 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-02T04:46:37.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:37 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:46:37.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:37 smithi192 ceph-mon[25697]: Deploying daemon osd.2 on smithi086 2024-04-02T04:46:38.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:38 smithi086 ceph-mon[24811]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T04:46:38.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:38 smithi086 ceph-mon[29505]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T04:46:38.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:38 smithi192 ceph-mon[25697]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T04:46:41.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:41 smithi086 ceph-mon[24811]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T04:46:41.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:41 smithi086 ceph-mon[29505]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T04:46:41.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:41 smithi192 ceph-mon[25697]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T04:46:42.414 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:42 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:46:42.414 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:42 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:42.414 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:42 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:46:42.414 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:42 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:46:42.414 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:42 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:46:42.414 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:42 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:42.415 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:42 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:46:42.415 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:42 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:46:42.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:42 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:46:42.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:42 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:42.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:42 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:46:42.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:42 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:46:43.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:43 smithi086 ceph-mon[29505]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T04:46:43.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:43 smithi086 ceph-mon[24811]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T04:46:43.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:43 smithi192 ceph-mon[25697]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T04:46:44.156 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:44 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:46:44.156 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:44 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:46:44.156 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:44 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:46:44.156 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:44 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:46:44.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:44 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:46:44.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:44 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:46:45.055 INFO:teuthology.orchestra.run.smithi086.stdout:Created osd(s) 2 on host 'smithi086' 2024-04-02T04:46:45.389 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:45 smithi086 ceph-mon[29505]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T04:46:45.389 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:45 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:45.389 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:45 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:45.389 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:45 smithi086 ceph-mon[29505]: from='osd.2 [v2:172.21.15.86:6818/1835017224,v1:172.21.15.86:6819/1835017224]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-02T04:46:45.389 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:45 smithi086 ceph-mon[29505]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-02T04:46:45.389 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:45 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:45.389 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:45 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:46:45.389 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:45 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:46:45.389 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:45 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:46:45.390 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:45 smithi086 ceph-mon[24811]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T04:46:45.390 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:45 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:45.390 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:45 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:45.390 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:45 smithi086 ceph-mon[24811]: from='osd.2 [v2:172.21.15.86:6818/1835017224,v1:172.21.15.86:6819/1835017224]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-02T04:46:45.390 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:45 smithi086 ceph-mon[24811]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-02T04:46:45.390 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:45 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:45.390 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:45 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:46:45.390 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:45 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:46:45.390 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:45 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:46:45.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:45 smithi192 ceph-mon[25697]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T04:46:45.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:45 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:45.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:45 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:45.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:45 smithi192 ceph-mon[25697]: from='osd.2 [v2:172.21.15.86:6818/1835017224,v1:172.21.15.86:6819/1835017224]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-02T04:46:45.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:45 smithi192 ceph-mon[25697]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-02T04:46:45.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:45 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:45.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:45 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:46:45.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:45 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:46:45.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:45 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:46:45.628 DEBUG:teuthology.orchestra.run.smithi086:osd.2> sudo journalctl -f -n 0 -u ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.2.service 2024-04-02T04:46:45.630 INFO:tasks.cephadm:Deploying osd.3 on smithi086 with /dev/vg_nvme/lv_1... 2024-04-02T04:46:45.631 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-02T04:46:46.620 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:46 smithi086 ceph-mon[29505]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-02T04:46:46.620 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:46 smithi086 ceph-mon[29505]: osdmap e16: 3 total, 2 up, 3 in 2024-04-02T04:46:46.620 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:46 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T04:46:46.620 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:46 smithi086 ceph-mon[29505]: from='osd.2 [v2:172.21.15.86:6818/1835017224,v1:172.21.15.86:6819/1835017224]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T04:46:46.620 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:46 smithi086 ceph-mon[29505]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T04:46:46.620 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:46 smithi086 ceph-mon[29505]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T04:46:46.621 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:46 smithi086 ceph-mon[24811]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-02T04:46:46.621 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:46 smithi086 ceph-mon[24811]: osdmap e16: 3 total, 2 up, 3 in 2024-04-02T04:46:46.621 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:46 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T04:46:46.621 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:46 smithi086 ceph-mon[24811]: from='osd.2 [v2:172.21.15.86:6818/1835017224,v1:172.21.15.86:6819/1835017224]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T04:46:46.621 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:46 smithi086 ceph-mon[24811]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T04:46:46.621 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:46 smithi086 ceph-mon[24811]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T04:46:46.622 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 04:46:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2[38252]: 2024-04-02T04:46:46.371+0000 7f70ffc5f700 -1 osd.2 0 waiting for initial osdmap 2024-04-02T04:46:46.622 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 04:46:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2[38252]: 2024-04-02T04:46:46.374+0000 7f70f85f2700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T04:46:46.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:46 smithi192 ceph-mon[25697]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-02T04:46:46.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:46 smithi192 ceph-mon[25697]: osdmap e16: 3 total, 2 up, 3 in 2024-04-02T04:46:46.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:46 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T04:46:46.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:46 smithi192 ceph-mon[25697]: from='osd.2 [v2:172.21.15.86:6818/1835017224,v1:172.21.15.86:6819/1835017224]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T04:46:46.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:46 smithi192 ceph-mon[25697]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T04:46:46.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:46 smithi192 ceph-mon[25697]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T04:46:47.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:47 smithi086 ceph-mon[24811]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]': finished 2024-04-02T04:46:47.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:47 smithi086 ceph-mon[24811]: osdmap e17: 3 total, 2 up, 3 in 2024-04-02T04:46:47.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:47 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T04:46:47.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:47 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T04:46:47.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:47 smithi086 ceph-mon[29505]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]': finished 2024-04-02T04:46:47.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:47 smithi086 ceph-mon[29505]: osdmap e17: 3 total, 2 up, 3 in 2024-04-02T04:46:47.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:47 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T04:46:47.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:47 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T04:46:47.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:47 smithi192 ceph-mon[25697]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]': finished 2024-04-02T04:46:47.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:47 smithi192 ceph-mon[25697]: osdmap e17: 3 total, 2 up, 3 in 2024-04-02T04:46:47.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:47 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T04:46:47.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:47 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T04:46:48.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:48 smithi086 ceph-mon[24811]: purged_snaps scrub starts 2024-04-02T04:46:48.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:48 smithi086 ceph-mon[24811]: purged_snaps scrub ok 2024-04-02T04:46:48.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:48 smithi086 ceph-mon[24811]: osd.2 [v2:172.21.15.86:6818/1835017224,v1:172.21.15.86:6819/1835017224] boot 2024-04-02T04:46:48.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:48 smithi086 ceph-mon[24811]: osdmap e18: 3 total, 3 up, 3 in 2024-04-02T04:46:48.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:48 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T04:46:48.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:48 smithi086 ceph-mon[24811]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T04:46:48.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:48 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-02T04:46:48.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:48 smithi086 ceph-mon[29505]: purged_snaps scrub starts 2024-04-02T04:46:48.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:48 smithi086 ceph-mon[29505]: purged_snaps scrub ok 2024-04-02T04:46:48.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:48 smithi086 ceph-mon[29505]: osd.2 [v2:172.21.15.86:6818/1835017224,v1:172.21.15.86:6819/1835017224] boot 2024-04-02T04:46:48.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:48 smithi086 ceph-mon[29505]: osdmap e18: 3 total, 3 up, 3 in 2024-04-02T04:46:48.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:48 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T04:46:48.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:48 smithi086 ceph-mon[29505]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T04:46:48.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:48 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-02T04:46:48.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:48 smithi192 ceph-mon[25697]: purged_snaps scrub starts 2024-04-02T04:46:48.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:48 smithi192 ceph-mon[25697]: purged_snaps scrub ok 2024-04-02T04:46:48.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:48 smithi192 ceph-mon[25697]: osd.2 [v2:172.21.15.86:6818/1835017224,v1:172.21.15.86:6819/1835017224] boot 2024-04-02T04:46:48.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:48 smithi192 ceph-mon[25697]: osdmap e18: 3 total, 3 up, 3 in 2024-04-02T04:46:48.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:48 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T04:46:48.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:48 smithi192 ceph-mon[25697]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T04:46:48.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:48 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-02T04:46:49.329 INFO:teuthology.orchestra.run.smithi086.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-02T04:46:49.329 INFO:teuthology.orchestra.run.smithi086.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-02T04:46:49.329 INFO:teuthology.orchestra.run.smithi086.stderr: stderr: 10+0 records in 2024-04-02T04:46:49.329 INFO:teuthology.orchestra.run.smithi086.stderr:10+0 records out 2024-04-02T04:46:49.329 INFO:teuthology.orchestra.run.smithi086.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0123499 s, 849 MB/s 2024-04-02T04:46:49.329 INFO:teuthology.orchestra.run.smithi086.stderr:--> Zapping successful for: 2024-04-02T04:46:49.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:49 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-02T04:46:49.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:49 smithi086 ceph-mon[24811]: osdmap e19: 3 total, 3 up, 3 in 2024-04-02T04:46:49.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:49 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-02T04:46:49.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:49 smithi086 ceph-mon[24811]: Detected new or changed devices on smithi086 2024-04-02T04:46:49.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:49 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:49.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:49 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:46:49.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:49 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:46:49.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:49 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:46:49.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:49 smithi086 ceph-mon[24811]: Adjusting osd_memory_target on smithi086 to 5354M 2024-04-02T04:46:49.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:49 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:49.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:49 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:49.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:49 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-02T04:46:49.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:49 smithi086 ceph-mon[29505]: osdmap e19: 3 total, 3 up, 3 in 2024-04-02T04:46:49.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:49 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-02T04:46:49.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:49 smithi086 ceph-mon[29505]: Detected new or changed devices on smithi086 2024-04-02T04:46:49.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:49 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:49.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:49 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:46:49.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:49 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:46:49.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:49 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:46:49.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:49 smithi086 ceph-mon[29505]: Adjusting osd_memory_target on smithi086 to 5354M 2024-04-02T04:46:49.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:49 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:49.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:49 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:49.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:49 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-02T04:46:49.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:49 smithi192 ceph-mon[25697]: osdmap e19: 3 total, 3 up, 3 in 2024-04-02T04:46:49.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:49 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-02T04:46:49.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:49 smithi192 ceph-mon[25697]: Detected new or changed devices on smithi086 2024-04-02T04:46:49.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:49 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:49.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:49 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:46:49.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:49 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:46:49.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:49 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:46:49.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:49 smithi192 ceph-mon[25697]: Adjusting osd_memory_target on smithi086 to 5354M 2024-04-02T04:46:49.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:49 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:49.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:49 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:49.972 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 04:46:49 smithi086 sudo[39479]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-02T04:46:49.972 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 04:46:49 smithi086 sudo[39479]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T04:46:49.972 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 04:46:49 smithi086 sudo[39479]: pam_unix(sudo:session): session closed for user root 2024-04-02T04:46:50.096 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi086:vg_nvme/lv_1 2024-04-02T04:46:50.244 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 04:46:49 smithi086 sudo[39482]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-02T04:46:50.244 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 04:46:49 smithi086 sudo[39482]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T04:46:50.244 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 04:46:49 smithi086 sudo[39482]: pam_unix(sudo:session): session closed for user root 2024-04-02T04:46:50.504 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:50 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-02T04:46:50.504 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:50 smithi086 ceph-mon[29505]: osdmap e20: 3 total, 3 up, 3 in 2024-04-02T04:46:50.504 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:50 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:50.504 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:50 smithi086 ceph-mon[29505]: pgmap v73: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-02T04:46:50.504 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 04:46:50 smithi086 sudo[39503]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-02T04:46:50.504 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 04:46:50 smithi086 sudo[39503]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T04:46:50.505 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 04:46:50 smithi086 sudo[39503]: pam_unix(sudo:session): session closed for user root 2024-04-02T04:46:50.505 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:50 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-02T04:46:50.505 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:50 smithi086 ceph-mon[24811]: osdmap e20: 3 total, 3 up, 3 in 2024-04-02T04:46:50.505 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:50 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:50.505 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:50 smithi086 ceph-mon[24811]: pgmap v73: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-02T04:46:50.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:50 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-02T04:46:50.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:50 smithi192 ceph-mon[25697]: osdmap e20: 3 total, 3 up, 3 in 2024-04-02T04:46:50.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:50 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:50.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:50 smithi192 ceph-mon[25697]: pgmap v73: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-02T04:46:50.754 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 04:46:50 smithi086 sudo[39555]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-02T04:46:50.755 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 04:46:50 smithi086 sudo[39532]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-02T04:46:50.755 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 04:46:50 smithi086 sudo[39532]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T04:46:50.755 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 04:46:50 smithi086 sudo[39532]: pam_unix(sudo:session): session closed for user root 2024-04-02T04:46:51.011 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 04:46:50 smithi086 sudo[39555]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T04:46:51.011 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 04:46:50 smithi086 sudo[39555]: pam_unix(sudo:session): session closed for user root 2024-04-02T04:46:51.270 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:51 smithi086 sudo[39562]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-02T04:46:51.270 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:51 smithi086 sudo[39562]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T04:46:51.270 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:51 smithi086 sudo[39562]: pam_unix(sudo:session): session closed for user root 2024-04-02T04:46:51.270 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 04:46:51 smithi086 sudo[39558]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-02T04:46:51.270 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 04:46:51 smithi086 sudo[39558]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T04:46:51.270 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 04:46:51 smithi086 sudo[39558]: pam_unix(sudo:session): session closed for user root 2024-04-02T04:46:51.521 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:51 smithi086 sudo[39583]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-02T04:46:51.521 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:51 smithi086 sudo[39583]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T04:46:51.521 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:51 smithi086 sudo[39583]: pam_unix(sudo:session): session closed for user root 2024-04-02T04:46:51.780 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:51 smithi086 ceph-mon[24811]: osdmap e21: 3 total, 3 up, 3 in 2024-04-02T04:46:51.780 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:51 smithi086 ceph-mon[24811]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-02T04:46:51.780 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:51 smithi086 ceph-mon[24811]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-02T04:46:51.780 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:51 smithi086 ceph-mon[29505]: osdmap e21: 3 total, 3 up, 3 in 2024-04-02T04:46:51.780 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:51 smithi086 ceph-mon[29505]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-02T04:46:51.780 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:51 smithi086 ceph-mon[29505]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-02T04:46:51.780 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:51 smithi086 sudo[39593]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-02T04:46:51.780 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:51 smithi086 sudo[39593]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T04:46:51.780 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:51 smithi086 sudo[39593]: pam_unix(sudo:session): session closed for user root 2024-04-02T04:46:51.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:51 smithi192 ceph-mon[25697]: osdmap e21: 3 total, 3 up, 3 in 2024-04-02T04:46:51.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:51 smithi192 ceph-mon[25697]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-02T04:46:51.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:51 smithi192 ceph-mon[25697]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-02T04:46:52.065 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:52 smithi086 sudo[39616]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-02T04:46:52.065 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:52 smithi086 sudo[39616]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T04:46:52.065 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:52 smithi086 sudo[39616]: pam_unix(sudo:session): session closed for user root 2024-04-02T04:46:52.516 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:52 smithi192 sudo[27208]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-02T04:46:52.516 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:52 smithi192 sudo[27208]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T04:46:52.516 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:52 smithi192 sudo[27208]: pam_unix(sudo:session): session closed for user root 2024-04-02T04:46:52.516 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:52 smithi192 sudo[27211]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-02T04:46:52.516 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:52 smithi192 sudo[27211]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T04:46:52.516 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:52 smithi192 sudo[27211]: pam_unix(sudo:session): session closed for user root 2024-04-02T04:46:52.744 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[29505]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-02T04:46:52.744 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T04:46:52.744 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T04:46:52.744 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T04:46:52.744 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[29505]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-02T04:46:52.745 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T04:46:52.745 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T04:46:52.745 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T04:46:52.745 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[29505]: from='client.14307 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi086:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:46:52.745 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T04:46:52.745 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T04:46:52.745 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:46:52.745 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[29505]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-02T04:46:52.745 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[29505]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-02T04:46:52.745 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T04:46:52.745 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T04:46:52.745 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T04:46:52.745 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[29505]: pgmap v75: 1 pgs: 1 active+clean; 577 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-04-02T04:46:52.745 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[29505]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-02T04:46:52.746 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[24811]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-02T04:46:52.746 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T04:46:52.746 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T04:46:52.746 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T04:46:52.746 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[24811]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-02T04:46:52.746 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T04:46:52.746 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T04:46:52.746 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T04:46:52.746 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[24811]: from='client.14307 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi086:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:46:52.746 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T04:46:52.746 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T04:46:52.747 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:46:52.747 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[24811]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-02T04:46:52.747 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[24811]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-02T04:46:52.747 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T04:46:52.747 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T04:46:52.747 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T04:46:52.747 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[24811]: pgmap v75: 1 pgs: 1 active+clean; 577 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-04-02T04:46:52.747 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:52 smithi086 ceph-mon[24811]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-02T04:46:52.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:52 smithi192 ceph-mon[25697]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-02T04:46:52.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:52 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T04:46:52.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:52 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T04:46:52.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:52 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T04:46:52.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:52 smithi192 ceph-mon[25697]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-02T04:46:52.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:52 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T04:46:52.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:52 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T04:46:52.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:52 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T04:46:52.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:52 smithi192 ceph-mon[25697]: from='client.14307 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi086:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:46:52.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:52 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T04:46:52.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:52 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T04:46:52.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:52 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:46:52.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:52 smithi192 ceph-mon[25697]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-02T04:46:52.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:52 smithi192 ceph-mon[25697]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-02T04:46:52.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:52 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T04:46:52.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:52 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T04:46:52.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:52 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T04:46:52.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:52 smithi192 ceph-mon[25697]: pgmap v75: 1 pgs: 1 active+clean; 577 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-04-02T04:46:52.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:52 smithi192 ceph-mon[25697]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-02T04:46:53.670 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:53 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:53.670 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:53 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:53.830 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:53 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:46:54.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:54 smithi086 ceph-mon[29505]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-02T04:46:54.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:54 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/2804423858' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "58bbbf2d-749e-4534-8800-0e945099e4bc"}]: dispatch 2024-04-02T04:46:54.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:54 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/2804423858' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "58bbbf2d-749e-4534-8800-0e945099e4bc"}]': finished 2024-04-02T04:46:54.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:54 smithi086 ceph-mon[29505]: osdmap e22: 4 total, 3 up, 4 in 2024-04-02T04:46:54.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:54 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T04:46:54.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:54 smithi086 ceph-mon[24811]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-02T04:46:54.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:54 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/2804423858' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "58bbbf2d-749e-4534-8800-0e945099e4bc"}]: dispatch 2024-04-02T04:46:54.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:54 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/2804423858' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "58bbbf2d-749e-4534-8800-0e945099e4bc"}]': finished 2024-04-02T04:46:54.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:54 smithi086 ceph-mon[24811]: osdmap e22: 4 total, 3 up, 4 in 2024-04-02T04:46:54.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:54 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T04:46:54.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:54 smithi192 ceph-mon[25697]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-02T04:46:54.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:54 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/2804423858' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "58bbbf2d-749e-4534-8800-0e945099e4bc"}]: dispatch 2024-04-02T04:46:54.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:54 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/2804423858' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "58bbbf2d-749e-4534-8800-0e945099e4bc"}]': finished 2024-04-02T04:46:54.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:54 smithi192 ceph-mon[25697]: osdmap e22: 4 total, 3 up, 4 in 2024-04-02T04:46:54.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:54 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T04:46:55.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:55 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/2616215463' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T04:46:55.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:55 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/2616215463' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T04:46:55.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:55 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/2616215463' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T04:46:56.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:56 smithi192 ceph-mon[25697]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-04-02T04:46:57.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:56 smithi086 ceph-mon[24811]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-04-02T04:46:57.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:56 smithi086 ceph-mon[29505]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-04-02T04:46:59.374 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:46:59 smithi086 ceph-mon[29505]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-04-02T04:46:59.375 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:46:59 smithi086 ceph-mon[24811]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-04-02T04:46:59.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:46:59 smithi192 ceph-mon[25697]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-04-02T04:47:01.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:01 smithi192 ceph-mon[25697]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-04-02T04:47:01.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:01 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-02T04:47:01.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:01 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:47:01.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:01 smithi086 ceph-mon[24811]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-04-02T04:47:01.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:01 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-02T04:47:01.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:01 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:47:01.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:01 smithi086 ceph-mon[29505]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-04-02T04:47:01.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:01 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-02T04:47:01.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:01 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:47:02.274 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:02 smithi086 ceph-mon[29505]: Deploying daemon osd.3 on smithi086 2024-04-02T04:47:02.274 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:02 smithi086 ceph-mon[24811]: Deploying daemon osd.3 on smithi086 2024-04-02T04:47:02.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:02 smithi192 ceph-mon[25697]: Deploying daemon osd.3 on smithi086 2024-04-02T04:47:03.384 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:03 smithi086 ceph-mon[24811]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-02T04:47:03.385 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:03 smithi086 ceph-mon[29505]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-02T04:47:03.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:03 smithi192 ceph-mon[25697]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-02T04:47:05.249 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:05 smithi086 ceph-mon[29505]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-02T04:47:05.249 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:05 smithi086 ceph-mon[24811]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-02T04:47:05.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:05 smithi192 ceph-mon[25697]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-02T04:47:06.323 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:06 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:06.323 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:06 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:47:06.323 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:06 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:47:06.323 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:06 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:47:06.324 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:06 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:06.324 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:06 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:47:06.324 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:06 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:47:06.324 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:06 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:47:06.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:06 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:06.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:06 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:47:06.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:06 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:47:06.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:06 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:47:07.468 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:07 smithi086 ceph-mon[29505]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 67 KiB/s wr, 4 op/s 2024-04-02T04:47:07.469 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:07 smithi086 ceph-mon[24811]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 67 KiB/s wr, 4 op/s 2024-04-02T04:47:07.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:07 smithi192 ceph-mon[25697]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 67 KiB/s wr, 4 op/s 2024-04-02T04:47:08.524 INFO:teuthology.orchestra.run.smithi086.stdout:Created osd(s) 3 on host 'smithi086' 2024-04-02T04:47:09.163 DEBUG:teuthology.orchestra.run.smithi086:osd.3> sudo journalctl -f -n 0 -u ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.3.service 2024-04-02T04:47:09.166 INFO:tasks.cephadm:Deploying osd.4 on smithi192 with /dev/vg_nvme/lv_4... 2024-04-02T04:47:09.166 DEBUG:teuthology.orchestra.run.smithi192:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-02T04:47:09.483 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:09 smithi192 ceph-mon[25697]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-02T04:47:09.484 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:09 smithi192 ceph-mon[25697]: from='osd.3 [v2:172.21.15.86:6826/1252559528,v1:172.21.15.86:6827/1252559528]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-02T04:47:09.484 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:09 smithi192 ceph-mon[25697]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-02T04:47:09.484 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:09 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:09.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:09 smithi086 ceph-mon[24811]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-02T04:47:09.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:09 smithi086 ceph-mon[24811]: from='osd.3 [v2:172.21.15.86:6826/1252559528,v1:172.21.15.86:6827/1252559528]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-02T04:47:09.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:09 smithi086 ceph-mon[24811]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-02T04:47:09.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:09 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:09.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:09 smithi086 ceph-mon[29505]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-02T04:47:09.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:09 smithi086 ceph-mon[29505]: from='osd.3 [v2:172.21.15.86:6826/1252559528,v1:172.21.15.86:6827/1252559528]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-02T04:47:09.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:09 smithi086 ceph-mon[29505]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-02T04:47:09.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:09 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:10.266 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:10 smithi086 ceph-mon[24811]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-02T04:47:10.266 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:10 smithi086 ceph-mon[24811]: osdmap e23: 4 total, 3 up, 4 in 2024-04-02T04:47:10.266 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:10 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T04:47:10.266 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:10 smithi086 ceph-mon[24811]: from='osd.3 [v2:172.21.15.86:6826/1252559528,v1:172.21.15.86:6827/1252559528]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T04:47:10.266 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:10 smithi086 ceph-mon[24811]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T04:47:10.266 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:10 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:10.266 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:10 smithi086 ceph-mon[29505]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-02T04:47:10.267 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:10 smithi086 ceph-mon[29505]: osdmap e23: 4 total, 3 up, 4 in 2024-04-02T04:47:10.267 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:10 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T04:47:10.267 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:10 smithi086 ceph-mon[29505]: from='osd.3 [v2:172.21.15.86:6826/1252559528,v1:172.21.15.86:6827/1252559528]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T04:47:10.267 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:10 smithi086 ceph-mon[29505]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T04:47:10.267 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:10 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:10.267 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 04:47:10 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3[41227]: 2024-04-02T04:47:10.231+0000 7fe451441700 -1 osd.3 0 waiting for initial osdmap 2024-04-02T04:47:10.267 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 04:47:10 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3[41227]: 2024-04-02T04:47:10.235+0000 7fe449dd4700 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T04:47:10.602 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:10 smithi192 ceph-mon[25697]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-02T04:47:10.602 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:10 smithi192 ceph-mon[25697]: osdmap e23: 4 total, 3 up, 4 in 2024-04-02T04:47:10.602 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:10 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T04:47:10.602 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:10 smithi192 ceph-mon[25697]: from='osd.3 [v2:172.21.15.86:6826/1252559528,v1:172.21.15.86:6827/1252559528]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T04:47:10.603 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:10 smithi192 ceph-mon[25697]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T04:47:10.603 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:10 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:11.311 INFO:teuthology.orchestra.run.smithi192.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-02T04:47:11.312 INFO:teuthology.orchestra.run.smithi192.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-02T04:47:11.312 INFO:teuthology.orchestra.run.smithi192.stderr: stderr: 10+0 records in 2024-04-02T04:47:11.312 INFO:teuthology.orchestra.run.smithi192.stderr:10+0 records out 2024-04-02T04:47:11.312 INFO:teuthology.orchestra.run.smithi192.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0112892 s, 929 MB/s 2024-04-02T04:47:11.312 INFO:teuthology.orchestra.run.smithi192.stderr:--> Zapping successful for: 2024-04-02T04:47:11.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:11 smithi086 ceph-mon[24811]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-02T04:47:11.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:11 smithi086 ceph-mon[24811]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]': finished 2024-04-02T04:47:11.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:11 smithi086 ceph-mon[24811]: osdmap e24: 4 total, 3 up, 4 in 2024-04-02T04:47:11.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:11 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T04:47:11.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:11 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T04:47:11.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:11 smithi086 ceph-mon[29505]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-02T04:47:11.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:11 smithi086 ceph-mon[29505]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]': finished 2024-04-02T04:47:11.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:11 smithi086 ceph-mon[29505]: osdmap e24: 4 total, 3 up, 4 in 2024-04-02T04:47:11.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:11 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T04:47:11.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:11 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T04:47:11.589 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:11 smithi192 ceph-mon[25697]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-04-02T04:47:11.589 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:11 smithi192 ceph-mon[25697]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]': finished 2024-04-02T04:47:11.589 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:11 smithi192 ceph-mon[25697]: osdmap e24: 4 total, 3 up, 4 in 2024-04-02T04:47:11.589 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:11 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T04:47:11.589 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:11 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T04:47:11.889 DEBUG:teuthology.orchestra.run.smithi192:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi192:vg_nvme/lv_4 2024-04-02T04:47:12.452 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:12 smithi192 ceph-mon[25697]: purged_snaps scrub starts 2024-04-02T04:47:12.452 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:12 smithi192 ceph-mon[25697]: purged_snaps scrub ok 2024-04-02T04:47:12.452 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:12 smithi192 ceph-mon[25697]: osd.3 [v2:172.21.15.86:6826/1252559528,v1:172.21.15.86:6827/1252559528] boot 2024-04-02T04:47:12.452 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:12 smithi192 ceph-mon[25697]: osdmap e25: 4 total, 4 up, 4 in 2024-04-02T04:47:12.453 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:12 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T04:47:12.453 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:12 smithi192 ceph-mon[25697]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T04:47:12.467 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:12 smithi086 ceph-mon[29505]: purged_snaps scrub starts 2024-04-02T04:47:12.467 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:12 smithi086 ceph-mon[29505]: purged_snaps scrub ok 2024-04-02T04:47:12.467 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:12 smithi086 ceph-mon[29505]: osd.3 [v2:172.21.15.86:6826/1252559528,v1:172.21.15.86:6827/1252559528] boot 2024-04-02T04:47:12.467 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:12 smithi086 ceph-mon[29505]: osdmap e25: 4 total, 4 up, 4 in 2024-04-02T04:47:12.467 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:12 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T04:47:12.468 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:12 smithi086 ceph-mon[29505]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T04:47:12.468 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:12 smithi086 ceph-mon[24811]: purged_snaps scrub starts 2024-04-02T04:47:12.468 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:12 smithi086 ceph-mon[24811]: purged_snaps scrub ok 2024-04-02T04:47:12.468 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:12 smithi086 ceph-mon[24811]: osd.3 [v2:172.21.15.86:6826/1252559528,v1:172.21.15.86:6827/1252559528] boot 2024-04-02T04:47:12.468 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:12 smithi086 ceph-mon[24811]: osdmap e25: 4 total, 4 up, 4 in 2024-04-02T04:47:12.468 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:12 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T04:47:12.468 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:12 smithi086 ceph-mon[24811]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T04:47:13.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:13 smithi192 ceph-mon[25697]: osdmap e26: 4 total, 4 up, 4 in 2024-04-02T04:47:13.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:13 smithi192 ceph-mon[25697]: Detected new or changed devices on smithi086 2024-04-02T04:47:13.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:13 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:13.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:13 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:47:13.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:13 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:47:13.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:13 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:47:13.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:13 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:47:13.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:13 smithi192 ceph-mon[25697]: Adjusting osd_memory_target on smithi086 to 4016M 2024-04-02T04:47:13.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:13 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:13.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:13 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:13.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:13 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:47:13.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:13 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:47:13.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:13 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:47:13.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:13 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:13.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:13 smithi086 ceph-mon[24811]: osdmap e26: 4 total, 4 up, 4 in 2024-04-02T04:47:13.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:13 smithi086 ceph-mon[24811]: Detected new or changed devices on smithi086 2024-04-02T04:47:13.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:13 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:13.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:13 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:47:13.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:13 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:47:13.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:13 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:47:13.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:13 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:47:13.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:13 smithi086 ceph-mon[24811]: Adjusting osd_memory_target on smithi086 to 4016M 2024-04-02T04:47:13.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:13 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:13.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:13 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:13.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:13 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:47:13.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:13 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:47:13.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:13 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:47:13.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:13 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:13.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:13 smithi086 ceph-mon[29505]: osdmap e26: 4 total, 4 up, 4 in 2024-04-02T04:47:13.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:13 smithi086 ceph-mon[29505]: Detected new or changed devices on smithi086 2024-04-02T04:47:13.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:13 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:13.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:13 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:47:13.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:13 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:47:13.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:13 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:47:13.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:13 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:47:13.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:13 smithi086 ceph-mon[29505]: Adjusting osd_memory_target on smithi086 to 4016M 2024-04-02T04:47:13.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:13 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:13.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:13 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:13.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:13 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:47:13.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:13 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:47:13.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:13 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:47:13.586 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:13 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:14.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:14 smithi192 ceph-mon[25697]: from='client.34181 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi192:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:47:14.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:14 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T04:47:14.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:14 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T04:47:14.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:14 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:47:14.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:14 smithi192 ceph-mon[25697]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T04:47:14.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:14 smithi086 ceph-mon[24811]: from='client.34181 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi192:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:47:14.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:14 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T04:47:14.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:14 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T04:47:14.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:14 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:47:14.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:14 smithi086 ceph-mon[24811]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T04:47:14.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:14 smithi086 ceph-mon[29505]: from='client.34181 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi192:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:47:14.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T04:47:14.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T04:47:14.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:14 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:47:14.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:14 smithi086 ceph-mon[29505]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T04:47:16.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:16 smithi192 ceph-mon[25697]: from='client.? 172.21.15.192:0/3702534994' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b0021f30-c92c-43dc-898e-99dd4618ef4e"}]: dispatch 2024-04-02T04:47:16.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:16 smithi192 ceph-mon[25697]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b0021f30-c92c-43dc-898e-99dd4618ef4e"}]: dispatch 2024-04-02T04:47:16.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:16 smithi192 ceph-mon[25697]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b0021f30-c92c-43dc-898e-99dd4618ef4e"}]': finished 2024-04-02T04:47:16.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:16 smithi192 ceph-mon[25697]: osdmap e27: 5 total, 4 up, 5 in 2024-04-02T04:47:16.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:16 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T04:47:16.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:16 smithi192 ceph-mon[25697]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T04:47:16.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:16 smithi192 ceph-mon[25697]: from='client.? 172.21.15.192:0/2766428069' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T04:47:17.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:16 smithi086 ceph-mon[24811]: from='client.? 172.21.15.192:0/3702534994' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b0021f30-c92c-43dc-898e-99dd4618ef4e"}]: dispatch 2024-04-02T04:47:17.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:16 smithi086 ceph-mon[24811]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b0021f30-c92c-43dc-898e-99dd4618ef4e"}]: dispatch 2024-04-02T04:47:17.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:16 smithi086 ceph-mon[24811]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b0021f30-c92c-43dc-898e-99dd4618ef4e"}]': finished 2024-04-02T04:47:17.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:16 smithi086 ceph-mon[24811]: osdmap e27: 5 total, 4 up, 5 in 2024-04-02T04:47:17.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:16 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T04:47:17.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:16 smithi086 ceph-mon[24811]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T04:47:17.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:16 smithi086 ceph-mon[24811]: from='client.? 172.21.15.192:0/2766428069' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T04:47:17.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:16 smithi086 ceph-mon[29505]: from='client.? 172.21.15.192:0/3702534994' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b0021f30-c92c-43dc-898e-99dd4618ef4e"}]: dispatch 2024-04-02T04:47:17.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:16 smithi086 ceph-mon[29505]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b0021f30-c92c-43dc-898e-99dd4618ef4e"}]: dispatch 2024-04-02T04:47:17.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:16 smithi086 ceph-mon[29505]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b0021f30-c92c-43dc-898e-99dd4618ef4e"}]': finished 2024-04-02T04:47:17.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:16 smithi086 ceph-mon[29505]: osdmap e27: 5 total, 4 up, 5 in 2024-04-02T04:47:17.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:16 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T04:47:17.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:16 smithi086 ceph-mon[29505]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T04:47:17.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:16 smithi086 ceph-mon[29505]: from='client.? 172.21.15.192:0/2766428069' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T04:47:19.293 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:19 smithi192 ceph-mon[25697]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T04:47:19.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:19 smithi086 ceph-mon[24811]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T04:47:19.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:19 smithi086 ceph-mon[29505]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T04:47:21.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:21 smithi192 ceph-mon[25697]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T04:47:21.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:21 smithi086 ceph-mon[24811]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T04:47:21.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:21 smithi086 ceph-mon[29505]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T04:47:22.416 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:22 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-02T04:47:22.416 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:22 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:47:22.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:22 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-02T04:47:22.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:22 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:47:22.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:22 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-02T04:47:22.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:22 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:47:23.333 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:23 smithi192 ceph-mon[25697]: Deploying daemon osd.4 on smithi192 2024-04-02T04:47:23.333 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:23 smithi192 ceph-mon[25697]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T04:47:23.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:23 smithi086 ceph-mon[24811]: Deploying daemon osd.4 on smithi192 2024-04-02T04:47:23.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:23 smithi086 ceph-mon[24811]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T04:47:23.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:23 smithi086 ceph-mon[29505]: Deploying daemon osd.4 on smithi192 2024-04-02T04:47:23.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:23 smithi086 ceph-mon[29505]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T04:47:25.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:25 smithi192 ceph-mon[25697]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T04:47:25.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:25 smithi086 ceph-mon[24811]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T04:47:25.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:25 smithi086 ceph-mon[29505]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T04:47:26.807 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:26 smithi192 ceph-mon[25697]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T04:47:27.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:26 smithi086 ceph-mon[24811]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T04:47:27.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:26 smithi086 ceph-mon[29505]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T04:47:28.072 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:27 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:28.072 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:27 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:47:28.073 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:27 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:47:28.073 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:27 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:47:28.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:27 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:28.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:27 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:47:28.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:27 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:47:28.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:27 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:47:28.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:27 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:28.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:27 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:47:28.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:27 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:47:28.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:27 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:47:28.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:28 smithi192 ceph-mon[25697]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T04:47:29.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:28 smithi086 ceph-mon[24811]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T04:47:29.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:28 smithi086 ceph-mon[29505]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T04:47:29.279 INFO:teuthology.orchestra.run.smithi192.stdout:Created osd(s) 4 on host 'smithi192' 2024-04-02T04:47:30.148 DEBUG:teuthology.orchestra.run.smithi192:osd.4> sudo journalctl -f -n 0 -u ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.4.service 2024-04-02T04:47:30.150 INFO:tasks.cephadm:Deploying osd.5 on smithi192 with /dev/vg_nvme/lv_3... 2024-04-02T04:47:30.151 DEBUG:teuthology.orchestra.run.smithi192:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-02T04:47:30.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:30 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:30.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:30 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:30.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:30 smithi192 ceph-mon[25697]: from='osd.4 [v2:172.21.15.192:6800/2803467108,v1:172.21.15.192:6801/2803467108]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-02T04:47:30.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:30 smithi192 ceph-mon[25697]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-02T04:47:30.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:30 smithi192 ceph-mon[25697]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T04:47:30.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:30 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:30.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:30 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:30.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:30 smithi086 ceph-mon[24811]: from='osd.4 [v2:172.21.15.192:6800/2803467108,v1:172.21.15.192:6801/2803467108]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-02T04:47:30.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:30 smithi086 ceph-mon[24811]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-02T04:47:30.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:30 smithi086 ceph-mon[24811]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T04:47:30.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:30 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:30.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:30 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:30.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:30 smithi086 ceph-mon[29505]: from='osd.4 [v2:172.21.15.192:6800/2803467108,v1:172.21.15.192:6801/2803467108]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-02T04:47:30.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:30 smithi086 ceph-mon[29505]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-02T04:47:30.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:30 smithi086 ceph-mon[29505]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T04:47:31.812 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:31 smithi192 ceph-mon[25697]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-02T04:47:31.812 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:31 smithi192 ceph-mon[25697]: osdmap e28: 5 total, 4 up, 5 in 2024-04-02T04:47:31.812 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:31 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T04:47:31.812 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:31 smithi192 ceph-mon[25697]: from='osd.4 [v2:172.21.15.192:6800/2803467108,v1:172.21.15.192:6801/2803467108]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T04:47:31.812 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:31 smithi192 ceph-mon[25697]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T04:47:31.812 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 04:47:31 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4[29147]: 2024-04-02T04:47:31.659+0000 7f9b18cee700 -1 osd.4 0 waiting for initial osdmap 2024-04-02T04:47:31.812 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 04:47:31 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4[29147]: 2024-04-02T04:47:31.663+0000 7f9b14687700 -1 osd.4 29 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T04:47:32.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:31 smithi086 ceph-mon[24811]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-02T04:47:32.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:31 smithi086 ceph-mon[24811]: osdmap e28: 5 total, 4 up, 5 in 2024-04-02T04:47:32.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:31 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T04:47:32.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:31 smithi086 ceph-mon[24811]: from='osd.4 [v2:172.21.15.192:6800/2803467108,v1:172.21.15.192:6801/2803467108]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T04:47:32.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:31 smithi086 ceph-mon[24811]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T04:47:32.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:31 smithi086 ceph-mon[29505]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-02T04:47:32.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:31 smithi086 ceph-mon[29505]: osdmap e28: 5 total, 4 up, 5 in 2024-04-02T04:47:32.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:31 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T04:47:32.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:31 smithi086 ceph-mon[29505]: from='osd.4 [v2:172.21.15.192:6800/2803467108,v1:172.21.15.192:6801/2803467108]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T04:47:32.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:31 smithi086 ceph-mon[29505]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T04:47:32.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:32 smithi192 ceph-mon[25697]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]': finished 2024-04-02T04:47:32.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:32 smithi192 ceph-mon[25697]: osdmap e29: 5 total, 4 up, 5 in 2024-04-02T04:47:32.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:32 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T04:47:32.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:32 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T04:47:32.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:32 smithi192 ceph-mon[25697]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T04:47:32.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:32 smithi192 ceph-mon[25697]: osd.4 [v2:172.21.15.192:6800/2803467108,v1:172.21.15.192:6801/2803467108] boot 2024-04-02T04:47:32.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:32 smithi192 ceph-mon[25697]: osdmap e30: 5 total, 5 up, 5 in 2024-04-02T04:47:32.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:32 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T04:47:33.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:32 smithi086 ceph-mon[29505]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]': finished 2024-04-02T04:47:33.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:32 smithi086 ceph-mon[29505]: osdmap e29: 5 total, 4 up, 5 in 2024-04-02T04:47:33.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:32 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T04:47:33.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:32 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T04:47:33.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:32 smithi086 ceph-mon[29505]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T04:47:33.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:32 smithi086 ceph-mon[29505]: osd.4 [v2:172.21.15.192:6800/2803467108,v1:172.21.15.192:6801/2803467108] boot 2024-04-02T04:47:33.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:32 smithi086 ceph-mon[29505]: osdmap e30: 5 total, 5 up, 5 in 2024-04-02T04:47:33.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:32 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T04:47:33.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:32 smithi086 ceph-mon[24811]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]': finished 2024-04-02T04:47:33.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:32 smithi086 ceph-mon[24811]: osdmap e29: 5 total, 4 up, 5 in 2024-04-02T04:47:33.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:32 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T04:47:33.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:32 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T04:47:33.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:32 smithi086 ceph-mon[24811]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T04:47:33.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:32 smithi086 ceph-mon[24811]: osd.4 [v2:172.21.15.192:6800/2803467108,v1:172.21.15.192:6801/2803467108] boot 2024-04-02T04:47:33.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:32 smithi086 ceph-mon[24811]: osdmap e30: 5 total, 5 up, 5 in 2024-04-02T04:47:33.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:32 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T04:47:33.846 INFO:teuthology.orchestra.run.smithi192.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-02T04:47:33.847 INFO:teuthology.orchestra.run.smithi192.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-02T04:47:33.847 INFO:teuthology.orchestra.run.smithi192.stderr: stderr: 10+0 records in 2024-04-02T04:47:33.847 INFO:teuthology.orchestra.run.smithi192.stderr:10+0 records out 2024-04-02T04:47:33.847 INFO:teuthology.orchestra.run.smithi192.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0124422 s, 843 MB/s 2024-04-02T04:47:33.847 INFO:teuthology.orchestra.run.smithi192.stderr: stderr: 2024-04-02T04:47:33.847 INFO:teuthology.orchestra.run.smithi192.stderr:--> Zapping successful for: 2024-04-02T04:47:34.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:33 smithi192 ceph-mon[25697]: purged_snaps scrub starts 2024-04-02T04:47:34.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:33 smithi192 ceph-mon[25697]: purged_snaps scrub ok 2024-04-02T04:47:34.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:33 smithi192 ceph-mon[25697]: Detected new or changed devices on smithi192 2024-04-02T04:47:34.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:33 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:34.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:33 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:47:34.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:33 smithi192 ceph-mon[25697]: Adjusting osd_memory_target on smithi192 to 17088M 2024-04-02T04:47:34.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:33 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:34.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:33 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:34.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:33 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:47:34.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:33 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:47:34.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:33 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:47:34.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:33 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:34.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:33 smithi192 ceph-mon[25697]: osdmap e31: 5 total, 5 up, 5 in 2024-04-02T04:47:34.333 DEBUG:teuthology.orchestra.run.smithi192:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi192:vg_nvme/lv_3 2024-04-02T04:47:34.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:33 smithi086 ceph-mon[29505]: purged_snaps scrub starts 2024-04-02T04:47:34.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:33 smithi086 ceph-mon[29505]: purged_snaps scrub ok 2024-04-02T04:47:34.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:33 smithi086 ceph-mon[29505]: Detected new or changed devices on smithi192 2024-04-02T04:47:34.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:33 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:34.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:33 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:47:34.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:33 smithi086 ceph-mon[29505]: Adjusting osd_memory_target on smithi192 to 17088M 2024-04-02T04:47:34.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:33 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:34.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:33 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:34.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:33 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:47:34.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:33 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:47:34.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:33 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:47:34.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:33 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:34.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:33 smithi086 ceph-mon[29505]: osdmap e31: 5 total, 5 up, 5 in 2024-04-02T04:47:34.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:33 smithi086 ceph-mon[24811]: purged_snaps scrub starts 2024-04-02T04:47:34.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:33 smithi086 ceph-mon[24811]: purged_snaps scrub ok 2024-04-02T04:47:34.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:33 smithi086 ceph-mon[24811]: Detected new or changed devices on smithi192 2024-04-02T04:47:34.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:33 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:34.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:33 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:47:34.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:33 smithi086 ceph-mon[24811]: Adjusting osd_memory_target on smithi192 to 17088M 2024-04-02T04:47:34.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:33 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:34.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:33 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:34.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:33 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:47:34.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:33 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:47:34.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:33 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:47:34.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:33 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:34.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:33 smithi086 ceph-mon[24811]: osdmap e31: 5 total, 5 up, 5 in 2024-04-02T04:47:34.887 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:34 smithi192 ceph-mon[25697]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-02T04:47:34.887 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:34 smithi192 ceph-mon[25697]: osdmap e32: 5 total, 5 up, 5 in 2024-04-02T04:47:35.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:34 smithi086 ceph-mon[29505]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-02T04:47:35.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:34 smithi086 ceph-mon[29505]: osdmap e32: 5 total, 5 up, 5 in 2024-04-02T04:47:35.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:34 smithi086 ceph-mon[24811]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-02T04:47:35.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:34 smithi086 ceph-mon[24811]: osdmap e32: 5 total, 5 up, 5 in 2024-04-02T04:47:36.922 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:36 smithi192 ceph-mon[25697]: osdmap e33: 5 total, 5 up, 5 in 2024-04-02T04:47:36.922 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:36 smithi192 ceph-mon[25697]: pgmap v109: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-04-02T04:47:36.922 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:36 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T04:47:36.922 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:36 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T04:47:36.922 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:36 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:47:37.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:36 smithi086 ceph-mon[24811]: osdmap e33: 5 total, 5 up, 5 in 2024-04-02T04:47:37.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:36 smithi086 ceph-mon[24811]: pgmap v109: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-04-02T04:47:37.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:36 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T04:47:37.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:36 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T04:47:37.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:36 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:47:37.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:36 smithi086 ceph-mon[29505]: osdmap e33: 5 total, 5 up, 5 in 2024-04-02T04:47:37.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:36 smithi086 ceph-mon[29505]: pgmap v109: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-04-02T04:47:37.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:36 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T04:47:37.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:36 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T04:47:37.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:36 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:47:37.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:37 smithi192 ceph-mon[25697]: from='client.34208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi192:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:47:37.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:37 smithi192 ceph-mon[25697]: osdmap e34: 5 total, 5 up, 5 in 2024-04-02T04:47:38.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:37 smithi086 ceph-mon[29505]: from='client.34208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi192:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:47:38.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:37 smithi086 ceph-mon[29505]: osdmap e34: 5 total, 5 up, 5 in 2024-04-02T04:47:38.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:37 smithi086 ceph-mon[24811]: from='client.34208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi192:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:47:38.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:37 smithi086 ceph-mon[24811]: osdmap e34: 5 total, 5 up, 5 in 2024-04-02T04:47:38.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:38 smithi192 ceph-mon[25697]: pgmap v111: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 117 KiB/s, 0 objects/s recovering 2024-04-02T04:47:38.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:38 smithi192 ceph-mon[25697]: from='client.? 172.21.15.192:0/1878331539' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3246bff-975b-40b4-90fc-daa551484ec9"}]: dispatch 2024-04-02T04:47:38.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:38 smithi192 ceph-mon[25697]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3246bff-975b-40b4-90fc-daa551484ec9"}]: dispatch 2024-04-02T04:47:38.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:38 smithi192 ceph-mon[25697]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a3246bff-975b-40b4-90fc-daa551484ec9"}]': finished 2024-04-02T04:47:38.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:38 smithi192 ceph-mon[25697]: osdmap e35: 6 total, 5 up, 6 in 2024-04-02T04:47:38.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:38 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T04:47:39.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:38 smithi086 ceph-mon[24811]: pgmap v111: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 117 KiB/s, 0 objects/s recovering 2024-04-02T04:47:39.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:38 smithi086 ceph-mon[24811]: from='client.? 172.21.15.192:0/1878331539' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3246bff-975b-40b4-90fc-daa551484ec9"}]: dispatch 2024-04-02T04:47:39.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:38 smithi086 ceph-mon[24811]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3246bff-975b-40b4-90fc-daa551484ec9"}]: dispatch 2024-04-02T04:47:39.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:38 smithi086 ceph-mon[24811]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a3246bff-975b-40b4-90fc-daa551484ec9"}]': finished 2024-04-02T04:47:39.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:38 smithi086 ceph-mon[24811]: osdmap e35: 6 total, 5 up, 6 in 2024-04-02T04:47:39.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:38 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T04:47:39.087 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:38 smithi086 ceph-mon[29505]: pgmap v111: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 117 KiB/s, 0 objects/s recovering 2024-04-02T04:47:39.087 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:38 smithi086 ceph-mon[29505]: from='client.? 172.21.15.192:0/1878331539' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3246bff-975b-40b4-90fc-daa551484ec9"}]: dispatch 2024-04-02T04:47:39.087 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:38 smithi086 ceph-mon[29505]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3246bff-975b-40b4-90fc-daa551484ec9"}]: dispatch 2024-04-02T04:47:39.087 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:38 smithi086 ceph-mon[29505]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a3246bff-975b-40b4-90fc-daa551484ec9"}]': finished 2024-04-02T04:47:39.087 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:38 smithi086 ceph-mon[29505]: osdmap e35: 6 total, 5 up, 6 in 2024-04-02T04:47:39.087 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:38 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T04:47:39.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:39 smithi192 ceph-mon[25697]: from='client.? 172.21.15.192:0/293299682' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T04:47:40.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:39 smithi086 ceph-mon[24811]: from='client.? 172.21.15.192:0/293299682' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T04:47:40.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:39 smithi086 ceph-mon[29505]: from='client.? 172.21.15.192:0/293299682' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T04:47:40.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:40 smithi192 ceph-mon[25697]: pgmap v113: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 95 KiB/s, 0 objects/s recovering 2024-04-02T04:47:41.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:40 smithi086 ceph-mon[24811]: pgmap v113: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 95 KiB/s, 0 objects/s recovering 2024-04-02T04:47:41.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:40 smithi086 ceph-mon[29505]: pgmap v113: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 95 KiB/s, 0 objects/s recovering 2024-04-02T04:47:43.336 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:43 smithi192 ceph-mon[25697]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-04-02T04:47:43.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:43 smithi086 ceph-mon[24811]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-04-02T04:47:43.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:43 smithi086 ceph-mon[29505]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-04-02T04:47:44.337 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:44 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:47:44.338 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:44 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:47:44.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:44 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:47:44.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:44 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:47:44.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:44 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:47:44.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:44 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:47:45.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:45 smithi192 ceph-mon[25697]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T04:47:45.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:45 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-02T04:47:45.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:45 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:47:45.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:45 smithi086 ceph-mon[24811]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T04:47:45.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:45 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-02T04:47:45.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:45 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:47:45.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:45 smithi086 ceph-mon[29505]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T04:47:45.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:45 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-02T04:47:45.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:45 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:47:46.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:46 smithi192 ceph-mon[25697]: Deploying daemon osd.5 on smithi192 2024-04-02T04:47:46.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:46 smithi086 ceph-mon[29505]: Deploying daemon osd.5 on smithi192 2024-04-02T04:47:46.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:46 smithi086 ceph-mon[24811]: Deploying daemon osd.5 on smithi192 2024-04-02T04:47:47.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:47 smithi192 ceph-mon[25697]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T04:47:47.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:47 smithi086 ceph-mon[24811]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T04:47:47.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:47 smithi086 ceph-mon[29505]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T04:47:49.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:49 smithi192 ceph-mon[25697]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T04:47:49.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:49 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:49.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:49 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:47:49.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:49 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:47:49.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:49 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:47:49.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:49 smithi086 ceph-mon[29505]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T04:47:49.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:49 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:49.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:49 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:47:49.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:49 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:47:49.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:49 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:47:49.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:49 smithi086 ceph-mon[24811]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T04:47:49.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:49 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:49.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:49 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:47:49.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:49 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:47:49.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:49 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:47:51.260 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:51 smithi192 ceph-mon[25697]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T04:47:51.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:51 smithi086 ceph-mon[29505]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T04:47:51.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:51 smithi086 ceph-mon[24811]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T04:47:52.262 INFO:teuthology.orchestra.run.smithi192.stdout:Created osd(s) 5 on host 'smithi192' 2024-04-02T04:47:52.576 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:52 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:52.576 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:52 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:52.576 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:52 smithi192 ceph-mon[25697]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T04:47:52.576 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:52 smithi192 ceph-mon[25697]: from='osd.5 [v2:172.21.15.192:6808/1908092348,v1:172.21.15.192:6809/1908092348]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-02T04:47:52.576 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:52 smithi192 ceph-mon[25697]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-02T04:47:52.576 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:52 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:52.576 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:52 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:47:52.576 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:52 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:47:52.576 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:52 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:47:52.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:52 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:52.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:52 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:52.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:52 smithi086 ceph-mon[29505]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T04:47:52.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:52 smithi086 ceph-mon[29505]: from='osd.5 [v2:172.21.15.192:6808/1908092348,v1:172.21.15.192:6809/1908092348]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-02T04:47:52.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:52 smithi086 ceph-mon[29505]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-02T04:47:52.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:52 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:52.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:52 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:47:52.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:52 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:47:52.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:52 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:47:52.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:52 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:52.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:52 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:52.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:52 smithi086 ceph-mon[24811]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T04:47:52.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:52 smithi086 ceph-mon[24811]: from='osd.5 [v2:172.21.15.192:6808/1908092348,v1:172.21.15.192:6809/1908092348]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-02T04:47:52.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:52 smithi086 ceph-mon[24811]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-02T04:47:52.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:52 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:52.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:52 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:47:52.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:52 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:47:52.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:52 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:47:52.850 DEBUG:teuthology.orchestra.run.smithi192:osd.5> sudo journalctl -f -n 0 -u ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.5.service 2024-04-02T04:47:52.852 INFO:tasks.cephadm:Deploying osd.6 on smithi192 with /dev/vg_nvme/lv_2... 2024-04-02T04:47:52.852 DEBUG:teuthology.orchestra.run.smithi192:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-02T04:47:53.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:53 smithi192 ceph-mon[25697]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-02T04:47:53.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:53 smithi192 ceph-mon[25697]: osdmap e36: 6 total, 5 up, 6 in 2024-04-02T04:47:53.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:53 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T04:47:53.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:53 smithi192 ceph-mon[25697]: from='osd.5 [v2:172.21.15.192:6808/1908092348,v1:172.21.15.192:6809/1908092348]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T04:47:53.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:53 smithi192 ceph-mon[25697]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T04:47:53.679 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 04:47:53 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5[32021]: 2024-04-02T04:47:53.366+0000 7f4f307a9700 -1 osd.5 0 waiting for initial osdmap 2024-04-02T04:47:53.679 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 04:47:53 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5[32021]: 2024-04-02T04:47:53.370+0000 7f4f2b140700 -1 osd.5 37 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T04:47:53.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:53 smithi086 ceph-mon[24811]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-02T04:47:53.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:53 smithi086 ceph-mon[24811]: osdmap e36: 6 total, 5 up, 6 in 2024-04-02T04:47:53.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:53 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T04:47:53.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:53 smithi086 ceph-mon[24811]: from='osd.5 [v2:172.21.15.192:6808/1908092348,v1:172.21.15.192:6809/1908092348]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T04:47:53.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:53 smithi086 ceph-mon[24811]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T04:47:53.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:53 smithi086 ceph-mon[29505]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-02T04:47:53.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:53 smithi086 ceph-mon[29505]: osdmap e36: 6 total, 5 up, 6 in 2024-04-02T04:47:53.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:53 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T04:47:53.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:53 smithi086 ceph-mon[29505]: from='osd.5 [v2:172.21.15.192:6808/1908092348,v1:172.21.15.192:6809/1908092348]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T04:47:53.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:53 smithi086 ceph-mon[29505]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T04:47:54.673 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:54 smithi192 ceph-mon[25697]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]': finished 2024-04-02T04:47:54.673 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:54 smithi192 ceph-mon[25697]: osdmap e37: 6 total, 5 up, 6 in 2024-04-02T04:47:54.673 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:54 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T04:47:54.674 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:54 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T04:47:54.674 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:54 smithi192 ceph-mon[25697]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T04:47:54.674 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:54 smithi192 ceph-mon[25697]: osd.5 [v2:172.21.15.192:6808/1908092348,v1:172.21.15.192:6809/1908092348] boot 2024-04-02T04:47:54.674 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:54 smithi192 ceph-mon[25697]: osdmap e38: 6 total, 6 up, 6 in 2024-04-02T04:47:54.674 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:54 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T04:47:54.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:54 smithi086 ceph-mon[24811]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]': finished 2024-04-02T04:47:54.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:54 smithi086 ceph-mon[24811]: osdmap e37: 6 total, 5 up, 6 in 2024-04-02T04:47:54.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:54 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T04:47:54.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:54 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T04:47:54.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:54 smithi086 ceph-mon[24811]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T04:47:54.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:54 smithi086 ceph-mon[24811]: osd.5 [v2:172.21.15.192:6808/1908092348,v1:172.21.15.192:6809/1908092348] boot 2024-04-02T04:47:54.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:54 smithi086 ceph-mon[24811]: osdmap e38: 6 total, 6 up, 6 in 2024-04-02T04:47:54.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:54 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T04:47:54.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:54 smithi086 ceph-mon[29505]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]': finished 2024-04-02T04:47:54.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:54 smithi086 ceph-mon[29505]: osdmap e37: 6 total, 5 up, 6 in 2024-04-02T04:47:54.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:54 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T04:47:54.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:54 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T04:47:54.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:54 smithi086 ceph-mon[29505]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T04:47:54.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:54 smithi086 ceph-mon[29505]: osd.5 [v2:172.21.15.192:6808/1908092348,v1:172.21.15.192:6809/1908092348] boot 2024-04-02T04:47:54.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:54 smithi086 ceph-mon[29505]: osdmap e38: 6 total, 6 up, 6 in 2024-04-02T04:47:54.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:54 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T04:47:55.403 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:55 smithi192 ceph-mon[25697]: purged_snaps scrub starts 2024-04-02T04:47:55.403 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:55 smithi192 ceph-mon[25697]: purged_snaps scrub ok 2024-04-02T04:47:55.403 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:55 smithi192 ceph-mon[25697]: osdmap e39: 6 total, 6 up, 6 in 2024-04-02T04:47:55.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:55 smithi086 ceph-mon[24811]: purged_snaps scrub starts 2024-04-02T04:47:55.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:55 smithi086 ceph-mon[24811]: purged_snaps scrub ok 2024-04-02T04:47:55.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:55 smithi086 ceph-mon[24811]: osdmap e39: 6 total, 6 up, 6 in 2024-04-02T04:47:55.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:55 smithi086 ceph-mon[29505]: purged_snaps scrub starts 2024-04-02T04:47:55.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:55 smithi086 ceph-mon[29505]: purged_snaps scrub ok 2024-04-02T04:47:55.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:55 smithi086 ceph-mon[29505]: osdmap e39: 6 total, 6 up, 6 in 2024-04-02T04:47:56.443 INFO:teuthology.orchestra.run.smithi192.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-02T04:47:56.443 INFO:teuthology.orchestra.run.smithi192.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-02T04:47:56.444 INFO:teuthology.orchestra.run.smithi192.stderr: stderr: 10+0 records in 2024-04-02T04:47:56.444 INFO:teuthology.orchestra.run.smithi192.stderr:10+0 records out 2024-04-02T04:47:56.444 INFO:teuthology.orchestra.run.smithi192.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0143212 s, 732 MB/s 2024-04-02T04:47:56.444 INFO:teuthology.orchestra.run.smithi192.stderr:--> Zapping successful for: 2024-04-02T04:47:56.737 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:56 smithi192 ceph-mon[25697]: Detected new or changed devices on smithi192 2024-04-02T04:47:56.738 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:56 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:56.738 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:56 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:47:56.738 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:56 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:47:56.738 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:56 smithi192 ceph-mon[25697]: Adjusting osd_memory_target on smithi192 to 8544M 2024-04-02T04:47:56.738 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:56 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:56.738 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:56 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:56.738 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:56 smithi192 ceph-mon[25697]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-02T04:47:56.738 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:56 smithi192 ceph-mon[25697]: osdmap e40: 6 total, 6 up, 6 in 2024-04-02T04:47:56.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:56 smithi086 ceph-mon[24811]: Detected new or changed devices on smithi192 2024-04-02T04:47:56.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:56 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:56.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:56 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:47:56.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:56 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:47:56.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:56 smithi086 ceph-mon[24811]: Adjusting osd_memory_target on smithi192 to 8544M 2024-04-02T04:47:56.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:56 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:56.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:56 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:56.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:56 smithi086 ceph-mon[24811]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-02T04:47:56.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:56 smithi086 ceph-mon[24811]: osdmap e40: 6 total, 6 up, 6 in 2024-04-02T04:47:56.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:56 smithi086 ceph-mon[29505]: Detected new or changed devices on smithi192 2024-04-02T04:47:56.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:56 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:56.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:56 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:47:56.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:56 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:47:56.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:56 smithi086 ceph-mon[29505]: Adjusting osd_memory_target on smithi192 to 8544M 2024-04-02T04:47:56.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:56 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:56.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:56 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:47:56.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:56 smithi086 ceph-mon[29505]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-02T04:47:56.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:56 smithi086 ceph-mon[29505]: osdmap e40: 6 total, 6 up, 6 in 2024-04-02T04:47:57.046 DEBUG:teuthology.orchestra.run.smithi192:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi192:vg_nvme/lv_2 2024-04-02T04:47:58.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:58 smithi192 ceph-mon[25697]: osdmap e41: 6 total, 6 up, 6 in 2024-04-02T04:47:58.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:58 smithi192 ceph-mon[25697]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-02T04:47:59.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:58 smithi086 ceph-mon[24811]: osdmap e41: 6 total, 6 up, 6 in 2024-04-02T04:47:59.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:58 smithi086 ceph-mon[24811]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-02T04:47:59.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:58 smithi086 ceph-mon[29505]: osdmap e41: 6 total, 6 up, 6 in 2024-04-02T04:47:59.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:58 smithi086 ceph-mon[29505]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-02T04:47:59.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:59 smithi192 ceph-mon[25697]: osdmap e42: 6 total, 6 up, 6 in 2024-04-02T04:47:59.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:59 smithi192 ceph-mon[25697]: from='client.34235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi192:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:47:59.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:59 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T04:47:59.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:59 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T04:47:59.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:47:59 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:48:00.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:59 smithi086 ceph-mon[24811]: osdmap e42: 6 total, 6 up, 6 in 2024-04-02T04:48:00.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:59 smithi086 ceph-mon[24811]: from='client.34235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi192:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:48:00.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:59 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T04:48:00.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:59 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T04:48:00.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:47:59 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:48:00.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:59 smithi086 ceph-mon[29505]: osdmap e42: 6 total, 6 up, 6 in 2024-04-02T04:48:00.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:59 smithi086 ceph-mon[29505]: from='client.34235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi192:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:48:00.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:59 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T04:48:00.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:59 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T04:48:00.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:47:59 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:48:00.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:00 smithi192 ceph-mon[25697]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T04:48:01.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:00 smithi086 ceph-mon[29505]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T04:48:01.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:00 smithi086 ceph-mon[24811]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T04:48:01.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:01 smithi192 ceph-mon[25697]: from='client.? 172.21.15.192:0/1603747046' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b4eab3ce-dfc9-41d9-95bd-98135ed757a3"}]: dispatch 2024-04-02T04:48:01.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:01 smithi192 ceph-mon[25697]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b4eab3ce-dfc9-41d9-95bd-98135ed757a3"}]: dispatch 2024-04-02T04:48:01.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:01 smithi192 ceph-mon[25697]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b4eab3ce-dfc9-41d9-95bd-98135ed757a3"}]': finished 2024-04-02T04:48:01.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:01 smithi192 ceph-mon[25697]: osdmap e43: 7 total, 6 up, 7 in 2024-04-02T04:48:01.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:01 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T04:48:02.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:01 smithi086 ceph-mon[24811]: from='client.? 172.21.15.192:0/1603747046' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b4eab3ce-dfc9-41d9-95bd-98135ed757a3"}]: dispatch 2024-04-02T04:48:02.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:01 smithi086 ceph-mon[24811]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b4eab3ce-dfc9-41d9-95bd-98135ed757a3"}]: dispatch 2024-04-02T04:48:02.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:01 smithi086 ceph-mon[24811]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b4eab3ce-dfc9-41d9-95bd-98135ed757a3"}]': finished 2024-04-02T04:48:02.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:01 smithi086 ceph-mon[24811]: osdmap e43: 7 total, 6 up, 7 in 2024-04-02T04:48:02.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:01 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T04:48:02.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:01 smithi086 ceph-mon[29505]: from='client.? 172.21.15.192:0/1603747046' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b4eab3ce-dfc9-41d9-95bd-98135ed757a3"}]: dispatch 2024-04-02T04:48:02.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:01 smithi086 ceph-mon[29505]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b4eab3ce-dfc9-41d9-95bd-98135ed757a3"}]: dispatch 2024-04-02T04:48:02.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:01 smithi086 ceph-mon[29505]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b4eab3ce-dfc9-41d9-95bd-98135ed757a3"}]': finished 2024-04-02T04:48:02.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:01 smithi086 ceph-mon[29505]: osdmap e43: 7 total, 6 up, 7 in 2024-04-02T04:48:02.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:01 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T04:48:02.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:02 smithi192 ceph-mon[25697]: from='client.? 172.21.15.192:0/879697819' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T04:48:02.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:02 smithi192 ceph-mon[25697]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T04:48:03.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:02 smithi086 ceph-mon[29505]: from='client.? 172.21.15.192:0/879697819' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T04:48:03.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:02 smithi086 ceph-mon[29505]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T04:48:03.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:02 smithi086 ceph-mon[24811]: from='client.? 172.21.15.192:0/879697819' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T04:48:03.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:02 smithi086 ceph-mon[24811]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T04:48:05.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:05 smithi192 ceph-mon[25697]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T04:48:05.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:05 smithi086 ceph-mon[24811]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T04:48:05.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:05 smithi086 ceph-mon[29505]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T04:48:06.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:06 smithi192 ceph-mon[25697]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T04:48:07.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:06 smithi086 ceph-mon[24811]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T04:48:07.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:06 smithi086 ceph-mon[29505]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T04:48:07.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:07 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-02T04:48:07.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:07 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:48:08.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:07 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-02T04:48:08.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:07 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:48:08.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:07 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-02T04:48:08.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:07 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:48:08.658 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:08 smithi192 ceph-mon[25697]: Deploying daemon osd.6 on smithi192 2024-04-02T04:48:08.658 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:08 smithi192 ceph-mon[25697]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T04:48:09.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:08 smithi086 ceph-mon[24811]: Deploying daemon osd.6 on smithi192 2024-04-02T04:48:09.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:08 smithi086 ceph-mon[24811]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T04:48:09.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:08 smithi086 ceph-mon[29505]: Deploying daemon osd.6 on smithi192 2024-04-02T04:48:09.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:08 smithi086 ceph-mon[29505]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T04:48:11.409 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:11 smithi192 ceph-mon[25697]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T04:48:11.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:11 smithi086 ceph-mon[24811]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T04:48:11.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:11 smithi086 ceph-mon[29505]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T04:48:13.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:13 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:13.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:13 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:48:13.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:13 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:48:13.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:13 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:48:13.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:13 smithi086 ceph-mon[24811]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T04:48:13.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:13 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:13.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:13 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:48:13.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:13 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:48:13.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:13 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:48:13.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:13 smithi086 ceph-mon[29505]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T04:48:13.353 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:13 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:13.353 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:13 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:48:13.353 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:13 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:48:13.353 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:13 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:48:13.353 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:13 smithi192 ceph-mon[25697]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T04:48:15.131 INFO:teuthology.orchestra.run.smithi192.stdout:Created osd(s) 6 on host 'smithi192' 2024-04-02T04:48:15.340 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:15 smithi192 ceph-mon[25697]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T04:48:15.340 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:15 smithi192 ceph-mon[25697]: from='osd.6 [v2:172.21.15.192:6816/2019998998,v1:172.21.15.192:6817/2019998998]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-02T04:48:15.340 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:15 smithi192 ceph-mon[25697]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-02T04:48:15.340 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:15 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:15.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:15 smithi086 ceph-mon[24811]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T04:48:15.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:15 smithi086 ceph-mon[24811]: from='osd.6 [v2:172.21.15.192:6816/2019998998,v1:172.21.15.192:6817/2019998998]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-02T04:48:15.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:15 smithi086 ceph-mon[24811]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-02T04:48:15.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:15 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:15.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:15 smithi086 ceph-mon[29505]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T04:48:15.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:15 smithi086 ceph-mon[29505]: from='osd.6 [v2:172.21.15.192:6816/2019998998,v1:172.21.15.192:6817/2019998998]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-02T04:48:15.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:15 smithi086 ceph-mon[29505]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-02T04:48:15.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:15 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:15.740 DEBUG:teuthology.orchestra.run.smithi192:osd.6> sudo journalctl -f -n 0 -u ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.6.service 2024-04-02T04:48:15.743 INFO:tasks.cephadm:Deploying osd.7 on smithi192 with /dev/vg_nvme/lv_1... 2024-04-02T04:48:15.743 DEBUG:teuthology.orchestra.run.smithi192:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-02T04:48:16.245 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:16 smithi192 ceph-mon[25697]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-02T04:48:16.246 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:16 smithi192 ceph-mon[25697]: osdmap e44: 7 total, 6 up, 7 in 2024-04-02T04:48:16.246 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:16 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T04:48:16.246 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:16 smithi192 ceph-mon[25697]: from='osd.6 [v2:172.21.15.192:6816/2019998998,v1:172.21.15.192:6817/2019998998]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T04:48:16.246 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:16 smithi192 ceph-mon[25697]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T04:48:16.246 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:16 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:16.246 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 04:48:16 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[34906]: 2024-04-02T04:48:16.147+0000 7f2cd38ed700 -1 osd.6 0 waiting for initial osdmap 2024-04-02T04:48:16.246 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 04:48:16 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[34906]: 2024-04-02T04:48:16.155+0000 7f2cce284700 -1 osd.6 45 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T04:48:16.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:16 smithi086 ceph-mon[29505]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-02T04:48:16.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:16 smithi086 ceph-mon[29505]: osdmap e44: 7 total, 6 up, 7 in 2024-04-02T04:48:16.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:16 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T04:48:16.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:16 smithi086 ceph-mon[29505]: from='osd.6 [v2:172.21.15.192:6816/2019998998,v1:172.21.15.192:6817/2019998998]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T04:48:16.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:16 smithi086 ceph-mon[29505]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T04:48:16.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:16 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:16.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:16 smithi086 ceph-mon[24811]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-02T04:48:16.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:16 smithi086 ceph-mon[24811]: osdmap e44: 7 total, 6 up, 7 in 2024-04-02T04:48:16.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:16 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T04:48:16.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:16 smithi086 ceph-mon[24811]: from='osd.6 [v2:172.21.15.192:6816/2019998998,v1:172.21.15.192:6817/2019998998]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T04:48:16.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:16 smithi086 ceph-mon[24811]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T04:48:16.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:16 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:17.419 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:17 smithi192 ceph-mon[25697]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T04:48:17.419 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:17 smithi192 ceph-mon[25697]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]': finished 2024-04-02T04:48:17.419 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:17 smithi192 ceph-mon[25697]: osdmap e45: 7 total, 6 up, 7 in 2024-04-02T04:48:17.420 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:17 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T04:48:17.420 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:17 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T04:48:17.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:17 smithi086 ceph-mon[24811]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T04:48:17.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:17 smithi086 ceph-mon[24811]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]': finished 2024-04-02T04:48:17.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:17 smithi086 ceph-mon[24811]: osdmap e45: 7 total, 6 up, 7 in 2024-04-02T04:48:17.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:17 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T04:48:17.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:17 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T04:48:17.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:17 smithi086 ceph-mon[29505]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T04:48:17.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:17 smithi086 ceph-mon[29505]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]': finished 2024-04-02T04:48:17.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:17 smithi086 ceph-mon[29505]: osdmap e45: 7 total, 6 up, 7 in 2024-04-02T04:48:17.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:17 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T04:48:17.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:17 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T04:48:18.178 INFO:teuthology.orchestra.run.smithi192.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-02T04:48:18.178 INFO:teuthology.orchestra.run.smithi192.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-02T04:48:18.178 INFO:teuthology.orchestra.run.smithi192.stderr: stderr: 10+0 records in 2024-04-02T04:48:18.178 INFO:teuthology.orchestra.run.smithi192.stderr:10+0 records out 2024-04-02T04:48:18.179 INFO:teuthology.orchestra.run.smithi192.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0121137 s, 866 MB/s 2024-04-02T04:48:18.179 INFO:teuthology.orchestra.run.smithi192.stderr:--> Zapping successful for: 2024-04-02T04:48:18.305 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:18 smithi192 ceph-mon[25697]: purged_snaps scrub starts 2024-04-02T04:48:18.305 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:18 smithi192 ceph-mon[25697]: purged_snaps scrub ok 2024-04-02T04:48:18.305 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:18 smithi192 ceph-mon[25697]: osd.6 [v2:172.21.15.192:6816/2019998998,v1:172.21.15.192:6817/2019998998] boot 2024-04-02T04:48:18.306 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:18 smithi192 ceph-mon[25697]: osdmap e46: 7 total, 7 up, 7 in 2024-04-02T04:48:18.306 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:18 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T04:48:18.306 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:18 smithi192 ceph-mon[25697]: osdmap e47: 7 total, 7 up, 7 in 2024-04-02T04:48:18.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:18 smithi086 ceph-mon[24811]: purged_snaps scrub starts 2024-04-02T04:48:18.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:18 smithi086 ceph-mon[24811]: purged_snaps scrub ok 2024-04-02T04:48:18.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:18 smithi086 ceph-mon[24811]: osd.6 [v2:172.21.15.192:6816/2019998998,v1:172.21.15.192:6817/2019998998] boot 2024-04-02T04:48:18.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:18 smithi086 ceph-mon[24811]: osdmap e46: 7 total, 7 up, 7 in 2024-04-02T04:48:18.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:18 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T04:48:18.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:18 smithi086 ceph-mon[24811]: osdmap e47: 7 total, 7 up, 7 in 2024-04-02T04:48:18.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:18 smithi086 ceph-mon[29505]: purged_snaps scrub starts 2024-04-02T04:48:18.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:18 smithi086 ceph-mon[29505]: purged_snaps scrub ok 2024-04-02T04:48:18.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:18 smithi086 ceph-mon[29505]: osd.6 [v2:172.21.15.192:6816/2019998998,v1:172.21.15.192:6817/2019998998] boot 2024-04-02T04:48:18.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:18 smithi086 ceph-mon[29505]: osdmap e46: 7 total, 7 up, 7 in 2024-04-02T04:48:18.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:18 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T04:48:18.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:18 smithi086 ceph-mon[29505]: osdmap e47: 7 total, 7 up, 7 in 2024-04-02T04:48:18.872 DEBUG:teuthology.orchestra.run.smithi192:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi192:vg_nvme/lv_1 2024-04-02T04:48:19.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:19 smithi192 ceph-mon[25697]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-02T04:48:19.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:19 smithi192 ceph-mon[25697]: osdmap e48: 7 total, 7 up, 7 in 2024-04-02T04:48:19.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:19 smithi086 ceph-mon[29505]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-02T04:48:19.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:19 smithi086 ceph-mon[29505]: osdmap e48: 7 total, 7 up, 7 in 2024-04-02T04:48:19.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:19 smithi086 ceph-mon[24811]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-02T04:48:19.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:19 smithi086 ceph-mon[24811]: osdmap e48: 7 total, 7 up, 7 in 2024-04-02T04:48:21.440 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:21 smithi192 ceph-mon[25697]: pgmap v146: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-02T04:48:21.440 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:21 smithi192 ceph-mon[25697]: osdmap e49: 7 total, 7 up, 7 in 2024-04-02T04:48:21.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:21 smithi086 ceph-mon[29505]: pgmap v146: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-02T04:48:21.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:21 smithi086 ceph-mon[29505]: osdmap e49: 7 total, 7 up, 7 in 2024-04-02T04:48:21.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:21 smithi086 ceph-mon[24811]: pgmap v146: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-02T04:48:21.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:21 smithi086 ceph-mon[24811]: osdmap e49: 7 total, 7 up, 7 in 2024-04-02T04:48:22.370 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:22 smithi192 ceph-mon[25697]: osdmap e50: 7 total, 7 up, 7 in 2024-04-02T04:48:22.370 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:22 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:22.370 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:22 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:48:22.370 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:22 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:48:22.371 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:22 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:48:22.371 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:22 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:22.371 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:22 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:22.371 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:22 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:48:22.371 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:22 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:48:22.371 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:22 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:48:22.371 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:22 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:22.371 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:22 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T04:48:22.371 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:22 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T04:48:22.371 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:22 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:48:22.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:22 smithi086 ceph-mon[29505]: osdmap e50: 7 total, 7 up, 7 in 2024-04-02T04:48:22.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:22 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:22.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:22 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:48:22.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:22 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:48:22.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:22 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:48:22.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:22 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:22.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:22 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:22.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:22 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:48:22.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:22 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:48:22.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:22 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:48:22.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:22 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:22.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:22 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T04:48:22.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:22 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T04:48:22.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:22 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:48:22.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:22 smithi086 ceph-mon[24811]: osdmap e50: 7 total, 7 up, 7 in 2024-04-02T04:48:22.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:22 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:22.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:22 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:48:22.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:22 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:48:22.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:22 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:48:22.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:22 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:22.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:22 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:22.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:22 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:48:22.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:22 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:48:22.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:22 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:48:22.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:22 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:22.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:22 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T04:48:22.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:22 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T04:48:22.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:22 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:48:23.240 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:23 smithi192 ceph-mon[25697]: Detected new or changed devices on smithi192 2024-04-02T04:48:23.240 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:23 smithi192 ceph-mon[25697]: Adjusting osd_memory_target on smithi192 to 5696M 2024-04-02T04:48:23.241 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:23 smithi192 ceph-mon[25697]: from='client.34262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi192:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:48:23.241 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:23 smithi192 ceph-mon[25697]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-02T04:48:23.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:23 smithi086 ceph-mon[29505]: Detected new or changed devices on smithi192 2024-04-02T04:48:23.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:23 smithi086 ceph-mon[29505]: Adjusting osd_memory_target on smithi192 to 5696M 2024-04-02T04:48:23.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:23 smithi086 ceph-mon[29505]: from='client.34262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi192:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:48:23.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:23 smithi086 ceph-mon[29505]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-02T04:48:23.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:23 smithi086 ceph-mon[24811]: Detected new or changed devices on smithi192 2024-04-02T04:48:23.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:23 smithi086 ceph-mon[24811]: Adjusting osd_memory_target on smithi192 to 5696M 2024-04-02T04:48:23.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:23 smithi086 ceph-mon[24811]: from='client.34262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi192:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:48:23.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:23 smithi086 ceph-mon[24811]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-02T04:48:25.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:25 smithi086 ceph-mon[29505]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-04-02T04:48:25.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:25 smithi086 ceph-mon[29505]: from='client.? 172.21.15.192:0/26210628' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f1ddd0fd-cf2e-4abf-a682-9c686712b926"}]: dispatch 2024-04-02T04:48:25.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:25 smithi086 ceph-mon[29505]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f1ddd0fd-cf2e-4abf-a682-9c686712b926"}]: dispatch 2024-04-02T04:48:25.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:25 smithi086 ceph-mon[29505]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f1ddd0fd-cf2e-4abf-a682-9c686712b926"}]': finished 2024-04-02T04:48:25.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:25 smithi086 ceph-mon[29505]: osdmap e51: 8 total, 7 up, 8 in 2024-04-02T04:48:25.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:25 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T04:48:25.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:25 smithi086 ceph-mon[29505]: from='client.? 172.21.15.192:0/3643176895' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T04:48:25.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:25 smithi086 ceph-mon[24811]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-04-02T04:48:25.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:25 smithi086 ceph-mon[24811]: from='client.? 172.21.15.192:0/26210628' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f1ddd0fd-cf2e-4abf-a682-9c686712b926"}]: dispatch 2024-04-02T04:48:25.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:25 smithi086 ceph-mon[24811]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f1ddd0fd-cf2e-4abf-a682-9c686712b926"}]: dispatch 2024-04-02T04:48:25.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:25 smithi086 ceph-mon[24811]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f1ddd0fd-cf2e-4abf-a682-9c686712b926"}]': finished 2024-04-02T04:48:25.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:25 smithi086 ceph-mon[24811]: osdmap e51: 8 total, 7 up, 8 in 2024-04-02T04:48:25.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:25 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T04:48:25.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:25 smithi086 ceph-mon[24811]: from='client.? 172.21.15.192:0/3643176895' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T04:48:25.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:25 smithi192 ceph-mon[25697]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-04-02T04:48:25.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:25 smithi192 ceph-mon[25697]: from='client.? 172.21.15.192:0/26210628' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f1ddd0fd-cf2e-4abf-a682-9c686712b926"}]: dispatch 2024-04-02T04:48:25.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:25 smithi192 ceph-mon[25697]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f1ddd0fd-cf2e-4abf-a682-9c686712b926"}]: dispatch 2024-04-02T04:48:25.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:25 smithi192 ceph-mon[25697]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f1ddd0fd-cf2e-4abf-a682-9c686712b926"}]': finished 2024-04-02T04:48:25.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:25 smithi192 ceph-mon[25697]: osdmap e51: 8 total, 7 up, 8 in 2024-04-02T04:48:25.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:25 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T04:48:25.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:25 smithi192 ceph-mon[25697]: from='client.? 172.21.15.192:0/3643176895' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T04:48:26.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:26 smithi192 ceph-mon[25697]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-02T04:48:27.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:26 smithi086 ceph-mon[29505]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-02T04:48:27.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:26 smithi086 ceph-mon[24811]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-02T04:48:29.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:29 smithi192 ceph-mon[25697]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-02T04:48:29.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:29 smithi086 ceph-mon[29505]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-02T04:48:29.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:29 smithi086 ceph-mon[24811]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-02T04:48:31.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:31 smithi192 ceph-mon[25697]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-02T04:48:31.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:31 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-02T04:48:31.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:31 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:48:31.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:31 smithi192 ceph-mon[25697]: Deploying daemon osd.7 on smithi192 2024-04-02T04:48:31.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:31 smithi086 ceph-mon[29505]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-02T04:48:31.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:31 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-02T04:48:31.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:31 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:48:31.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:31 smithi086 ceph-mon[29505]: Deploying daemon osd.7 on smithi192 2024-04-02T04:48:31.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:31 smithi086 ceph-mon[24811]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-02T04:48:31.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:31 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-02T04:48:31.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:31 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:48:31.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:31 smithi086 ceph-mon[24811]: Deploying daemon osd.7 on smithi192 2024-04-02T04:48:33.413 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:33 smithi192 ceph-mon[25697]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-02T04:48:33.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:33 smithi086 ceph-mon[24811]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-02T04:48:33.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:33 smithi086 ceph-mon[29505]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-02T04:48:35.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:35 smithi192 ceph-mon[25697]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T04:48:35.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:35 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:35.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:35 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:48:35.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:35 smithi086 ceph-mon[24811]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T04:48:35.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:35 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:35.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:35 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:48:35.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:35 smithi086 ceph-mon[29505]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T04:48:35.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:35 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:35.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:35 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:48:36.416 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:36 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:48:36.416 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:36 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:48:36.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:36 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:48:36.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:36 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:48:36.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:36 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:48:36.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:36 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:48:37.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:37 smithi192 ceph-mon[25697]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T04:48:37.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:37 smithi086 ceph-mon[29505]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T04:48:37.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:37 smithi086 ceph-mon[24811]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T04:48:38.543 INFO:teuthology.orchestra.run.smithi192.stdout:Created osd(s) 7 on host 'smithi192' 2024-04-02T04:48:38.845 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:38 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:38.845 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:38 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:38.845 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:38 smithi192 ceph-mon[25697]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T04:48:38.845 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:38 smithi192 ceph-mon[25697]: from='osd.7 [v2:172.21.15.192:6824/2081381372,v1:172.21.15.192:6825/2081381372]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-02T04:48:38.845 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:38 smithi192 ceph-mon[25697]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-02T04:48:38.845 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:38 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:38.845 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:38 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:48:38.846 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:38 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:48:38.846 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:38 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:48:39.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:38 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:39.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:38 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:39.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:38 smithi086 ceph-mon[29505]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T04:48:39.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:38 smithi086 ceph-mon[29505]: from='osd.7 [v2:172.21.15.192:6824/2081381372,v1:172.21.15.192:6825/2081381372]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-02T04:48:39.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:38 smithi086 ceph-mon[29505]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-02T04:48:39.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:38 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:39.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:38 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:48:39.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:38 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:48:39.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:38 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:48:39.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:38 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:39.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:38 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:39.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:38 smithi086 ceph-mon[24811]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T04:48:39.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:38 smithi086 ceph-mon[24811]: from='osd.7 [v2:172.21.15.192:6824/2081381372,v1:172.21.15.192:6825/2081381372]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-02T04:48:39.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:38 smithi086 ceph-mon[24811]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-02T04:48:39.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:38 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:39.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:38 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:48:39.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:38 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:48:39.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:38 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:48:39.141 DEBUG:teuthology.orchestra.run.smithi192:osd.7> sudo journalctl -f -n 0 -u ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.7.service 2024-04-02T04:48:39.143 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-04-02T04:48:39.143 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph osd stat -f json 2024-04-02T04:48:39.766 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:39 smithi086 ceph-mon[24811]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-02T04:48:39.767 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:39 smithi086 ceph-mon[24811]: osdmap e52: 8 total, 7 up, 8 in 2024-04-02T04:48:39.767 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:39 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T04:48:39.767 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:39 smithi086 ceph-mon[24811]: from='osd.7 [v2:172.21.15.192:6824/2081381372,v1:172.21.15.192:6825/2081381372]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T04:48:39.767 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:39 smithi086 ceph-mon[24811]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T04:48:39.767 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:39 smithi086 ceph-mon[29505]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-02T04:48:39.767 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:39 smithi086 ceph-mon[29505]: osdmap e52: 8 total, 7 up, 8 in 2024-04-02T04:48:39.767 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:39 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T04:48:39.767 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:39 smithi086 ceph-mon[29505]: from='osd.7 [v2:172.21.15.192:6824/2081381372,v1:172.21.15.192:6825/2081381372]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T04:48:39.767 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:39 smithi086 ceph-mon[29505]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T04:48:39.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:39 smithi192 ceph-mon[25697]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-02T04:48:39.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:39 smithi192 ceph-mon[25697]: osdmap e52: 8 total, 7 up, 8 in 2024-04-02T04:48:39.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:39 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T04:48:39.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:39 smithi192 ceph-mon[25697]: from='osd.7 [v2:172.21.15.192:6824/2081381372,v1:172.21.15.192:6825/2081381372]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T04:48:39.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:39 smithi192 ceph-mon[25697]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T04:48:39.928 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 04:48:39 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[37891]: 2024-04-02T04:48:39.658+0000 7ff4fe104700 -1 osd.7 0 waiting for initial osdmap 2024-04-02T04:48:39.928 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 04:48:39 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[37891]: 2024-04-02T04:48:39.665+0000 7ff4f8a9b700 -1 osd.7 53 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T04:48:40.915 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-02T04:48:40.915 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:40 smithi086 ceph-mon[24811]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]': finished 2024-04-02T04:48:40.915 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:40 smithi086 ceph-mon[24811]: osdmap e53: 8 total, 7 up, 8 in 2024-04-02T04:48:40.915 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:40 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T04:48:40.915 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:40 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T04:48:40.915 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:40 smithi086 ceph-mon[24811]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T04:48:40.915 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:40 smithi086 ceph-mon[24811]: osd.7 [v2:172.21.15.192:6824/2081381372,v1:172.21.15.192:6825/2081381372] boot 2024-04-02T04:48:40.915 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:40 smithi086 ceph-mon[24811]: osdmap e54: 8 total, 8 up, 8 in 2024-04-02T04:48:40.915 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:40 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T04:48:40.916 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:40 smithi086 ceph-mon[29505]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]': finished 2024-04-02T04:48:40.916 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:40 smithi086 ceph-mon[29505]: osdmap e53: 8 total, 7 up, 8 in 2024-04-02T04:48:40.916 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:40 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T04:48:40.916 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:40 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T04:48:40.916 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:40 smithi086 ceph-mon[29505]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T04:48:40.916 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:40 smithi086 ceph-mon[29505]: osd.7 [v2:172.21.15.192:6824/2081381372,v1:172.21.15.192:6825/2081381372] boot 2024-04-02T04:48:40.916 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:40 smithi086 ceph-mon[29505]: osdmap e54: 8 total, 8 up, 8 in 2024-04-02T04:48:40.916 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:40 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T04:48:40.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:40 smithi192 ceph-mon[25697]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]': finished 2024-04-02T04:48:40.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:40 smithi192 ceph-mon[25697]: osdmap e53: 8 total, 7 up, 8 in 2024-04-02T04:48:40.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:40 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T04:48:40.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:40 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T04:48:40.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:40 smithi192 ceph-mon[25697]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T04:48:40.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:40 smithi192 ceph-mon[25697]: osd.7 [v2:172.21.15.192:6824/2081381372,v1:172.21.15.192:6825/2081381372] boot 2024-04-02T04:48:40.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:40 smithi192 ceph-mon[25697]: osdmap e54: 8 total, 8 up, 8 in 2024-04-02T04:48:40.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:40 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T04:48:41.647 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":54,"num_osds":8,"num_up_osds":8,"osd_up_since":1712033320,"num_in_osds":8,"osd_in_since":1712033304,"num_remapped_pgs":1} 2024-04-02T04:48:41.648 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-02T04:48:41.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:41 smithi086 ceph-mon[29505]: purged_snaps scrub starts 2024-04-02T04:48:41.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:41 smithi086 ceph-mon[29505]: purged_snaps scrub ok 2024-04-02T04:48:41.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:41 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/2065653550' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-02T04:48:41.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:41 smithi086 ceph-mon[29505]: osdmap e55: 8 total, 8 up, 8 in 2024-04-02T04:48:41.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:41 smithi086 ceph-mon[24811]: purged_snaps scrub starts 2024-04-02T04:48:41.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:41 smithi086 ceph-mon[24811]: purged_snaps scrub ok 2024-04-02T04:48:41.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:41 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/2065653550' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-02T04:48:41.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:41 smithi086 ceph-mon[24811]: osdmap e55: 8 total, 8 up, 8 in 2024-04-02T04:48:41.871 INFO:teuthology.orchestra.run.smithi086.stderr:Inferring config /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T04:48:41.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:41 smithi192 ceph-mon[25697]: purged_snaps scrub starts 2024-04-02T04:48:41.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:41 smithi192 ceph-mon[25697]: purged_snaps scrub ok 2024-04-02T04:48:41.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:41 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/2065653550' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-02T04:48:41.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:41 smithi192 ceph-mon[25697]: osdmap e55: 8 total, 8 up, 8 in 2024-04-02T04:48:43.190 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:42 smithi086 ceph-mon[24811]: Detected new or changed devices on smithi192 2024-04-02T04:48:43.190 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:42 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:43.191 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:42 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:48:43.191 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:42 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:48:43.191 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:42 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:48:43.191 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:42 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:48:43.191 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:42 smithi086 ceph-mon[24811]: Adjusting osd_memory_target on smithi192 to 4272M 2024-04-02T04:48:43.191 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:42 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:43.191 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:42 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:43.191 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:42 smithi086 ceph-mon[24811]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:48:43.191 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:42 smithi086 ceph-mon[24811]: osdmap e56: 8 total, 8 up, 8 in 2024-04-02T04:48:43.191 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:43 smithi086 ceph-mon[29505]: Detected new or changed devices on smithi192 2024-04-02T04:48:43.191 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:43 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:43.191 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:43 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:48:43.192 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:43 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:48:43.192 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:43 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:48:43.192 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:43 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:48:43.192 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:43 smithi086 ceph-mon[29505]: Adjusting osd_memory_target on smithi192 to 4272M 2024-04-02T04:48:43.192 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:43 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:43.192 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:43 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:43.192 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:43 smithi086 ceph-mon[29505]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:48:43.192 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:43 smithi086 ceph-mon[29505]: osdmap e56: 8 total, 8 up, 8 in 2024-04-02T04:48:43.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:42 smithi192 ceph-mon[25697]: Detected new or changed devices on smithi192 2024-04-02T04:48:43.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:42 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:43.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:42 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:48:43.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:42 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:48:43.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:42 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:48:43.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:42 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:48:43.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:42 smithi192 ceph-mon[25697]: Adjusting osd_memory_target on smithi192 to 4272M 2024-04-02T04:48:43.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:42 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:43.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:42 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:43.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:42 smithi192 ceph-mon[25697]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:48:43.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:42 smithi192 ceph-mon[25697]: osdmap e56: 8 total, 8 up, 8 in 2024-04-02T04:48:43.536 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-02T04:48:43.536 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":56,"fsid":"8862e8fe-f0ab-11ee-b647-cb9ed24678a4","created":"2024-04-02T04:43:51.159039+0000","modified":"2024-04-02T04:48:42.629942+0000","last_up_change":"2024-04-02T04:48:40.627055+0000","last_in_change":"2024-04-02T04:48:24.210591+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-02T04:46:48.351740+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"e6ee7c83-68a3-4d34-8cdc-21972cf722bf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":2621947859},{"type":"v1","addr":"172.21.15.86:6803","nonce":2621947859}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":2621947859},{"type":"v1","addr":"172.21.15.86:6805","nonce":2621947859}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2621947859},{"type":"v1","addr":"172.21.15.86:6809","nonce":2621947859}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":2621947859},{"type":"v1","addr":"172.21.15.86:6807","nonce":2621947859}]},"public_addr":"172.21.15.86:6803/2621947859","cluster_addr":"172.21.15.86:6805/2621947859","heartbeat_back_addr":"172.21.15.86:6809/2621947859","heartbeat_front_addr":"172.21.15.86:6807/2621947859","state":["exists","up"]},{"osd":1,"uuid":"fb117c77-5aff-44fc-bcfd-23805e94220d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6810","nonce":3495515130},{"type":"v1","addr":"172.21.15.86:6811","nonce":3495515130}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":3495515130},{"type":"v1","addr":"172.21.15.86:6813","nonce":3495515130}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":3495515130},{"type":"v1","addr":"172.21.15.86:6817","nonce":3495515130}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6814","nonce":3495515130},{"type":"v1","addr":"172.21.15.86:6815","nonce":3495515130}]},"public_addr":"172.21.15.86:6811/3495515130","cluster_addr":"172.21.15.86:6813/3495515130","heartbeat_back_addr":"172.21.15.86:6817/3495515130","heartbeat_front_addr":"172.21.15.86:6815/3495515130","state":["exists","up"]},{"osd":2,"uuid":"a0ee7998-0bb3-4b9d-b536-d305b6679aa6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6818","nonce":1835017224},{"type":"v1","addr":"172.21.15.86:6819","nonce":1835017224}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":1835017224},{"type":"v1","addr":"172.21.15.86:6821","nonce":1835017224}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":1835017224},{"type":"v1","addr":"172.21.15.86:6825","nonce":1835017224}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6822","nonce":1835017224},{"type":"v1","addr":"172.21.15.86:6823","nonce":1835017224}]},"public_addr":"172.21.15.86:6819/1835017224","cluster_addr":"172.21.15.86:6821/1835017224","heartbeat_back_addr":"172.21.15.86:6825/1835017224","heartbeat_front_addr":"172.21.15.86:6823/1835017224","state":["exists","up"]},{"osd":3,"uuid":"58bbbf2d-749e-4534-8800-0e945099e4bc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":1252559528},{"type":"v1","addr":"172.21.15.86:6827","nonce":1252559528}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":1252559528},{"type":"v1","addr":"172.21.15.86:6829","nonce":1252559528}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6832","nonce":1252559528},{"type":"v1","addr":"172.21.15.86:6833","nonce":1252559528}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":1252559528},{"type":"v1","addr":"172.21.15.86:6831","nonce":1252559528}]},"public_addr":"172.21.15.86:6827/1252559528","cluster_addr":"172.21.15.86:6829/1252559528","heartbeat_back_addr":"172.21.15.86:6833/1252559528","heartbeat_front_addr":"172.21.15.86:6831/1252559528","state":["exists","up"]},{"osd":4,"uuid":"b0021f30-c92c-43dc-898e-99dd4618ef4e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6800","nonce":2803467108},{"type":"v1","addr":"172.21.15.192:6801","nonce":2803467108}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6802","nonce":2803467108},{"type":"v1","addr":"172.21.15.192:6803","nonce":2803467108}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6806","nonce":2803467108},{"type":"v1","addr":"172.21.15.192:6807","nonce":2803467108}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6804","nonce":2803467108},{"type":"v1","addr":"172.21.15.192:6805","nonce":2803467108}]},"public_addr":"172.21.15.192:6801/2803467108","cluster_addr":"172.21.15.192:6803/2803467108","heartbeat_back_addr":"172.21.15.192:6807/2803467108","heartbeat_front_addr":"172.21.15.192:6805/2803467108","state":["exists","up"]},{"osd":5,"uuid":"a3246bff-975b-40b4-90fc-daa551484ec9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6808","nonce":1908092348},{"type":"v1","addr":"172.21.15.192:6809","nonce":1908092348}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6810","nonce":1908092348},{"type":"v1","addr":"172.21.15.192:6811","nonce":1908092348}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6814","nonce":1908092348},{"type":"v1","addr":"172.21.15.192:6815","nonce":1908092348}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6812","nonce":1908092348},{"type":"v1","addr":"172.21.15.192:6813","nonce":1908092348}]},"public_addr":"172.21.15.192:6809/1908092348","cluster_addr":"172.21.15.192:6811/1908092348","heartbeat_back_addr":"172.21.15.192:6815/1908092348","heartbeat_front_addr":"172.21.15.192:6813/1908092348","state":["exists","up"]},{"osd":6,"uuid":"b4eab3ce-dfc9-41d9-95bd-98135ed757a3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6816","nonce":2019998998},{"type":"v1","addr":"172.21.15.192:6817","nonce":2019998998}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6818","nonce":2019998998},{"type":"v1","addr":"172.21.15.192:6819","nonce":2019998998}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6822","nonce":2019998998},{"type":"v1","addr":"172.21.15.192:6823","nonce":2019998998}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6820","nonce":2019998998},{"type":"v1","addr":"172.21.15.192:6821","nonce":2019998998}]},"public_addr":"172.21.15.192:6817/2019998998","cluster_addr":"172.21.15.192:6819/2019998998","heartbeat_back_addr":"172.21.15.192:6823/2019998998","heartbeat_front_addr":"172.21.15.192:6821/2019998998","state":["exists","up"]},{"osd":7,"uuid":"f1ddd0fd-cf2e-4abf-a682-9c686712b926","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6824","nonce":2081381372},{"type":"v1","addr":"172.21.15.192:6825","nonce":2081381372}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6826","nonce":2081381372},{"type":"v1","addr":"172.21.15.192:6827","nonce":2081381372}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6830","nonce":2081381372},{"type":"v1","addr":"172.21.15.192:6831","nonce":2081381372}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6828","nonce":2081381372},{"type":"v1","addr":"172.21.15.192:6829","nonce":2081381372}]},"public_addr":"172.21.15.192:6825/2081381372","cluster_addr":"172.21.15.192:6827/2081381372","heartbeat_back_addr":"172.21.15.192:6831/2081381372","heartbeat_front_addr":"172.21.15.192:6829/2081381372","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:46:00.645602+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:46:22.956048+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:46:45.818580+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:47:09.397500+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:47:30.864808+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:47:53.134635+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:48:16.072372+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:48:39.225607+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[{"pgid":"1.0","osds":[0,6]}],"primary_temp":[],"blocklist":{"172.21.15.86:0/2885713909":"2024-04-03T04:44:43.987095+0000","172.21.15.86:0/2853453946":"2024-04-03T04:44:43.987095+0000","172.21.15.86:6801/3904718154":"2024-04-03T04:44:43.987095+0000","172.21.15.86:6800/3904718154":"2024-04-03T04:44:43.987095+0000","172.21.15.86:0/4053196327":"2024-04-03T04:44:22.479015+0000","172.21.15.86:0/1081336665":"2024-04-03T04:44:22.479015+0000","172.21.15.86:6800/1417258151":"2024-04-03T04:44:22.479015+0000","172.21.15.86:0/2634385848":"2024-04-03T04:44:43.987095+0000","172.21.15.86:0/1565660562":"2024-04-03T04:44:22.479015+0000","172.21.15.86:6801/1417258151":"2024-04-03T04:44:22.479015+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-02T04:48:44.240 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:44 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1175668213' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T04:48:44.240 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:44 smithi086 ceph-mon[29505]: osdmap e57: 8 total, 8 up, 8 in 2024-04-02T04:48:44.241 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:44 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:48:44.060+0000 7f653e2a0700 -1 calc_pg_upmaps abort due to max <= 0 2024-04-02T04:48:44.241 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:44 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1175668213' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T04:48:44.241 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:44 smithi086 ceph-mon[24811]: osdmap e57: 8 total, 8 up, 8 in 2024-04-02T04:48:44.268 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-02T04:46:48.351740+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-04-02T04:48:44.268 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph osd pool get .mgr pg_num 2024-04-02T04:48:44.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:44 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1175668213' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T04:48:44.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:44 smithi192 ceph-mon[25697]: osdmap e57: 8 total, 8 up, 8 in 2024-04-02T04:48:44.484 INFO:teuthology.orchestra.run.smithi086.stderr:Inferring config /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T04:48:45.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:45 smithi086 ceph-mon[24811]: pgmap v167: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-04-02T04:48:45.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:45 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:48:45.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:45 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:48:45.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:45 smithi086 ceph-mon[24811]: osdmap e58: 8 total, 8 up, 8 in 2024-04-02T04:48:45.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:45 smithi086 ceph-mon[29505]: pgmap v167: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-04-02T04:48:45.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:45 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:48:45.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:45 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:48:45.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:45 smithi086 ceph-mon[29505]: osdmap e58: 8 total, 8 up, 8 in 2024-04-02T04:48:45.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:45 smithi192 ceph-mon[25697]: pgmap v167: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-04-02T04:48:45.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:45 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:48:45.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:45 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:48:45.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:45 smithi192 ceph-mon[25697]: osdmap e58: 8 total, 8 up, 8 in 2024-04-02T04:48:46.301 INFO:teuthology.orchestra.run.smithi086.stdout:pg_num: 1 2024-04-02T04:48:46.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:46 smithi192 ceph-mon[25697]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 116 KiB/s, 0 objects/s recovering 2024-04-02T04:48:46.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:46 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/2994312913' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-02T04:48:46.937 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:46 smithi086 ceph-mon[29505]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 116 KiB/s, 0 objects/s recovering 2024-04-02T04:48:46.937 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:46 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/2994312913' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-02T04:48:46.937 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:46 smithi086 ceph-mon[24811]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 116 KiB/s, 0 objects/s recovering 2024-04-02T04:48:46.937 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:46 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/2994312913' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-02T04:48:46.960 INFO:tasks.cephadm:Adding prometheus.a on smithi192 2024-04-02T04:48:46.960 DEBUG:teuthology.orchestra.run.smithi192:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph orch apply prometheus '1;smithi192=a' 2024-04-02T04:48:48.846 INFO:teuthology.orchestra.run.smithi192.stdout:Scheduled prometheus update... 2024-04-02T04:48:49.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:49 smithi192 ceph-mon[25697]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-02T04:48:49.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:49 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:49.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:49 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:48:49.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:49 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:48:49.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:49 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:48:49.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:49 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:49.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:49 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-02T04:48:49.492 DEBUG:teuthology.orchestra.run.smithi192:prometheus.a> sudo journalctl -f -n 0 -u ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@prometheus.a.service 2024-04-02T04:48:49.494 INFO:tasks.cephadm:Adding node-exporter.a on smithi086 2024-04-02T04:48:49.495 INFO:tasks.cephadm:Adding node-exporter.b on smithi192 2024-04-02T04:48:49.495 DEBUG:teuthology.orchestra.run.smithi192:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph orch apply node-exporter '2;smithi086=a;smithi192=b' 2024-04-02T04:48:49.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:49 smithi086 ceph-mon[24811]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-02T04:48:49.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:49 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:49.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:49 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:48:49.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:49 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:48:49.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:49 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:48:49.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:49 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:49.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:49 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-02T04:48:49.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:49 smithi086 ceph-mon[29505]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-02T04:48:49.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:49 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:49.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:49 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:48:49.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:49 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:48:49.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:49 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:48:49.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:49 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' 2024-04-02T04:48:49.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:49 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-02T04:48:50.012 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:48:49 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: ignoring --setuser ceph since I am not root 2024-04-02T04:48:50.012 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:48:49 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: ignoring --setgroup ceph since I am not root 2024-04-02T04:48:50.012 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:48:49 smithi192 ceph-mgr[26885]: -- 172.21.15.192:0/1396496096 <== mon.2 v2:172.21.15.192:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55fd0f7f0340 con 0x55fd0f7c8800 2024-04-02T04:48:50.143 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:49 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ignoring --setuser ceph since I am not root 2024-04-02T04:48:50.143 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:49 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ignoring --setgroup ceph since I am not root 2024-04-02T04:48:50.143 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:49 smithi086 ceph-mgr[25044]: -- 172.21.15.86:0/2621919892 <== mon.0 v2:172.21.15.86:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55a19b048340 con 0x55a19b021000 2024-04-02T04:48:50.381 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:50 smithi192 ceph-mon[25697]: from='client.34298 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi192=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:48:50.381 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:50 smithi192 ceph-mon[25697]: Saving service prometheus spec with placement smithi192=a;count:1 2024-04-02T04:48:50.381 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:50 smithi192 ceph-mon[25697]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-02T04:48:50.381 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:50 smithi192 ceph-mon[25697]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-02T04:48:50.437 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:50 smithi086 ceph-mon[29505]: from='client.34298 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi192=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:48:50.437 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:50 smithi086 ceph-mon[29505]: Saving service prometheus spec with placement smithi192=a;count:1 2024-04-02T04:48:50.437 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:50 smithi086 ceph-mon[29505]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-02T04:48:50.438 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:50 smithi086 ceph-mon[29505]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-02T04:48:50.438 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:50 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:48:50.346+0000 7f40a30c9000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-02T04:48:50.438 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:50 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:48:50.433+0000 7f40a30c9000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-02T04:48:50.438 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:50 smithi086 ceph-mon[24811]: from='client.34298 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi192=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:48:50.438 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:50 smithi086 ceph-mon[24811]: Saving service prometheus spec with placement smithi192=a;count:1 2024-04-02T04:48:50.438 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:50 smithi086 ceph-mon[24811]: from='mgr.14150 172.21.15.86:0/815204442' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-02T04:48:50.438 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:50 smithi086 ceph-mon[24811]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-02T04:48:50.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:48:50 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:48:50.377+0000 7fec4e84f000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-02T04:48:50.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:48:50 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:48:50.469+0000 7fec4e84f000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-02T04:48:51.333 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:51 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:48:51.020+0000 7f40a30c9000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-02T04:48:51.333 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:51 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:48:51.109+0000 7f40a30c9000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-02T04:48:51.333 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:51 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:48:51.190+0000 7f40a30c9000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-02T04:48:51.421 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:48:51 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:48:51.082+0000 7fec4e84f000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-02T04:48:51.422 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:48:51 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:48:51.174+0000 7fec4e84f000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-02T04:48:51.422 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:48:51 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:48:51.264+0000 7fec4e84f000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-02T04:48:51.611 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:51 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:48:51.339+0000 7f40a30c9000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-02T04:48:51.611 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:51 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:48:51.420+0000 7f40a30c9000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-02T04:48:51.611 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:51 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:48:51.526+0000 7f40a30c9000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-02T04:48:51.611 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:51 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:48:51.607+0000 7f40a30c9000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-02T04:48:51.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:48:51 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:48:51.417+0000 7fec4e84f000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-02T04:48:51.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:48:51 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:48:51.497+0000 7fec4e84f000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-02T04:48:51.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:48:51 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:48:51.603+0000 7fec4e84f000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-02T04:48:52.083 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:51 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:48:51.769+0000 7f40a30c9000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-02T04:48:52.083 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:51 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:48:51.856+0000 7f40a30c9000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-02T04:48:52.178 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:48:51 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:48:51.686+0000 7fec4e84f000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-02T04:48:52.178 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:48:51 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:48:51.848+0000 7fec4e84f000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-02T04:48:52.178 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:48:51 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:48:51.932+0000 7fec4e84f000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-02T04:48:53.189 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:52 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:48:52.867+0000 7f40a30c9000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-02T04:48:53.189 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:52 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:48:52.996+0000 7f40a30c9000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-02T04:48:53.189 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:53 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:48:53.185+0000 7f40a30c9000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-02T04:48:53.272 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:48:52 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:48:52.950+0000 7fec4e84f000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-02T04:48:53.272 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:48:53 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:48:53.079+0000 7fec4e84f000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-02T04:48:53.272 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:48:53 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:48:53.269+0000 7fec4e84f000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-02T04:48:53.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:53 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:48:53.273+0000 7f40a30c9000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-02T04:48:53.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:53 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:48:53.406+0000 7f40a30c9000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-02T04:48:53.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:48:53 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:48:53.357+0000 7fec4e84f000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-02T04:48:53.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:48:53 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:48:53.492+0000 7fec4e84f000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-02T04:48:54.033 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:53 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:48:53.723+0000 7f40a30c9000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-02T04:48:54.033 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:53 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:48:53.945+0000 7f40a30c9000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-02T04:48:54.088 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:48:53 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:48:53.809+0000 7fec4e84f000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-02T04:48:54.088 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:48:54 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:48:54.027+0000 7fec4e84f000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-02T04:48:54.319 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:54 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:48:54.029+0000 7f40a30c9000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-02T04:48:54.320 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:54 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:48:54.200+0000 7f40a30c9000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-02T04:48:54.427 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:48:54 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:48:54.112+0000 7fec4e84f000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-02T04:48:54.428 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:48:54 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:48:54.281+0000 7fec4e84f000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-02T04:48:55.434 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:55 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:48:55.135+0000 7f40a30c9000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-02T04:48:55.481 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:48:55 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:48:55.188+0000 7fec4e84f000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-02T04:48:55.830 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:48:55 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:48:55.478+0000 7fec4e84f000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-02T04:48:55.831 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:48:55 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:48:55.572+0000 7fec4e84f000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-02T04:48:55.831 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:48:55 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:48:55.720+0000 7fec4e84f000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-02T04:48:55.831 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:48:55 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: [02/Apr/2024:04:48:55] ENGINE Bus STARTING 2024-04-02T04:48:55.831 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:48:55 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: CherryPy Checker: 2024-04-02T04:48:55.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:55 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:48:55.431+0000 7f40a30c9000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-02T04:48:55.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:55 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:48:55.527+0000 7f40a30c9000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-02T04:48:55.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:55 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:48:55.680+0000 7f40a30c9000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-02T04:48:56.178 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:48:55 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: The Application mounted at '' has an empty config. 2024-04-02T04:48:56.178 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:48:55 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: 2024-04-02T04:48:56.178 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:48:55 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: [02/Apr/2024:04:48:55] ENGINE Serving on http://:::9283 2024-04-02T04:48:56.178 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:48:55 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: [02/Apr/2024:04:48:55] ENGINE Bus STARTED 2024-04-02T04:48:56.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:56 smithi192 ceph-mon[25697]: Active manager daemon y restarted 2024-04-02T04:48:56.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:56 smithi192 ceph-mon[25697]: Activating manager daemon y 2024-04-02T04:48:56.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:56 smithi192 ceph-mon[25697]: osdmap e59: 8 total, 8 up, 8 in 2024-04-02T04:48:56.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:56 smithi192 ceph-mon[25697]: Standby manager daemon x restarted 2024-04-02T04:48:56.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:56 smithi192 ceph-mon[25697]: Standby manager daemon x started 2024-04-02T04:48:56.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:56 smithi192 ceph-mon[25697]: from='mgr.? 172.21.15.192:0/2637338034' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-02T04:48:56.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:56 smithi192 ceph-mon[25697]: from='mgr.? 172.21.15.192:0/2637338034' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T04:48:56.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:56 smithi192 ceph-mon[25697]: from='mgr.? 172.21.15.192:0/2637338034' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-02T04:48:56.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:56 smithi192 ceph-mon[25697]: from='mgr.? 172.21.15.192:0/2637338034' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T04:48:56.972 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:56 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: [02/Apr/2024:04:48:56] ENGINE Bus STARTING 2024-04-02T04:48:56.973 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:56 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: CherryPy Checker: 2024-04-02T04:48:56.973 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:56 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: The Application mounted at '' has an empty config. 2024-04-02T04:48:56.973 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:56 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:48:56.973 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:56 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: [02/Apr/2024:04:48:56] ENGINE Serving on http://:::9283 2024-04-02T04:48:56.973 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:56 smithi086 ceph-mon[29505]: Active manager daemon y restarted 2024-04-02T04:48:56.973 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:56 smithi086 ceph-mon[29505]: Activating manager daemon y 2024-04-02T04:48:56.973 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:56 smithi086 ceph-mon[29505]: osdmap e59: 8 total, 8 up, 8 in 2024-04-02T04:48:56.973 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:56 smithi086 ceph-mon[29505]: Standby manager daemon x restarted 2024-04-02T04:48:56.973 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:56 smithi086 ceph-mon[29505]: Standby manager daemon x started 2024-04-02T04:48:56.974 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:56 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.192:0/2637338034' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-02T04:48:56.974 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:56 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.192:0/2637338034' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T04:48:56.974 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:56 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.192:0/2637338034' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-02T04:48:56.974 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:56 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.192:0/2637338034' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T04:48:56.974 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:56 smithi086 ceph-mon[24811]: Active manager daemon y restarted 2024-04-02T04:48:56.974 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:56 smithi086 ceph-mon[24811]: Activating manager daemon y 2024-04-02T04:48:56.974 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:56 smithi086 ceph-mon[24811]: osdmap e59: 8 total, 8 up, 8 in 2024-04-02T04:48:56.974 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:56 smithi086 ceph-mon[24811]: Standby manager daemon x restarted 2024-04-02T04:48:56.974 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:56 smithi086 ceph-mon[24811]: Standby manager daemon x started 2024-04-02T04:48:56.974 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:56 smithi086 ceph-mon[24811]: from='mgr.? 172.21.15.192:0/2637338034' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-02T04:48:56.974 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:56 smithi086 ceph-mon[24811]: from='mgr.? 172.21.15.192:0/2637338034' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T04:48:56.975 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:56 smithi086 ceph-mon[24811]: from='mgr.? 172.21.15.192:0/2637338034' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-02T04:48:56.975 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:56 smithi086 ceph-mon[24811]: from='mgr.? 172.21.15.192:0/2637338034' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T04:48:57.333 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:56 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: [02/Apr/2024:04:48:56] ENGINE Bus STARTED 2024-04-02T04:48:57.333 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: [02/Apr/2024:04:48:57] ENGINE Bus STARTING 2024-04-02T04:48:57.333 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: [02/Apr/2024:04:48:57] ENGINE Serving on https://172.21.15.86:7150 2024-04-02T04:48:57.334 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: [02/Apr/2024:04:48:57] ENGINE Bus STARTED 2024-04-02T04:48:57.671 INFO:teuthology.orchestra.run.smithi192.stdout:Scheduled node-exporter update... 2024-04-02T04:48:57.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:57 smithi192 ceph-mon[25697]: mgrmap e17: y(active, starting, since 0.962617s), standbys: x 2024-04-02T04:48:57.954 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:57 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T04:48:57.954 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:57 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T04:48:57.954 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:57 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T04:48:57.954 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:57 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T04:48:57.954 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:57 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T04:48:57.955 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:57 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T04:48:57.955 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:57 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T04:48:57.955 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:57 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T04:48:57.955 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:57 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T04:48:57.955 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:57 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T04:48:57.955 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:57 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T04:48:57.955 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:57 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T04:48:57.955 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:57 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T04:48:57.955 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:57 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T04:48:57.955 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:57 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T04:48:57.955 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:57 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T04:48:57.955 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:57 smithi192 ceph-mon[25697]: Manager daemon y is now available 2024-04-02T04:48:57.955 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:57 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:48:57.955 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:57 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:48:57.955 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:57 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:48:57.955 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:57 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:48:57.956 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:57 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:48:57.956 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:57 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:48:57.956 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:57 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:48:57.956 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:57 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:48:58.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[29505]: mgrmap e17: y(active, starting, since 0.962617s), standbys: x 2024-04-02T04:48:58.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T04:48:58.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T04:48:58.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T04:48:58.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T04:48:58.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T04:48:58.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T04:48:58.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T04:48:58.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T04:48:58.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T04:48:58.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T04:48:58.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T04:48:58.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T04:48:58.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T04:48:58.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T04:48:58.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T04:48:58.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T04:48:58.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[29505]: Manager daemon y is now available 2024-04-02T04:48:58.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:48:58.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:48:58.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:48:58.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:48:58.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:48:58.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:48:58.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:48:58.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:48:58.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[24811]: mgrmap e17: y(active, starting, since 0.962617s), standbys: x 2024-04-02T04:48:58.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T04:48:58.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T04:48:58.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T04:48:58.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T04:48:58.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T04:48:58.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T04:48:58.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T04:48:58.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T04:48:58.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T04:48:58.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T04:48:58.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T04:48:58.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T04:48:58.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T04:48:58.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T04:48:58.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T04:48:58.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T04:48:58.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[24811]: Manager daemon y is now available 2024-04-02T04:48:58.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:48:58.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:48:58.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:48:58.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:48:58.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:48:58.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:48:58.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:48:58.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:57 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:48:58.502 DEBUG:teuthology.orchestra.run.smithi086:node-exporter.a> sudo journalctl -f -n 0 -u ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@node-exporter.a.service 2024-04-02T04:48:58.504 DEBUG:teuthology.orchestra.run.smithi192:node-exporter.b> sudo journalctl -f -n 0 -u ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@node-exporter.b.service 2024-04-02T04:48:58.506 INFO:tasks.cephadm:Adding alertmanager.a on smithi086 2024-04-02T04:48:58.507 DEBUG:teuthology.orchestra.run.smithi192:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph orch apply alertmanager '1;smithi086=a' 2024-04-02T04:48:58.891 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:58 smithi192 ceph-mon[25697]: [02/Apr/2024:04:48:57] ENGINE Bus STARTING 2024-04-02T04:48:58.891 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:58 smithi192 ceph-mon[25697]: [02/Apr/2024:04:48:57] ENGINE Serving on https://172.21.15.86:7150 2024-04-02T04:48:58.891 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:58 smithi192 ceph-mon[25697]: [02/Apr/2024:04:48:57] ENGINE Bus STARTED 2024-04-02T04:48:58.891 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:58 smithi192 ceph-mon[25697]: mgrmap e18: y(active, since 1.97448s), standbys: x 2024-04-02T04:48:58.891 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:58 smithi192 ceph-mon[25697]: from='client.34316 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi086=a;smithi192=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:48:58.892 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:58 smithi192 ceph-mon[25697]: Saving service node-exporter spec with placement smithi086=a;smithi192=b;count:2 2024-04-02T04:48:58.892 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:58 smithi192 ceph-mon[25697]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:48:58.892 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:58 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:48:58.892 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:58 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:48:58.920 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:58 smithi086 ceph-mon[29505]: [02/Apr/2024:04:48:57] ENGINE Bus STARTING 2024-04-02T04:48:58.920 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:58 smithi086 ceph-mon[29505]: [02/Apr/2024:04:48:57] ENGINE Serving on https://172.21.15.86:7150 2024-04-02T04:48:58.920 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:58 smithi086 ceph-mon[29505]: [02/Apr/2024:04:48:57] ENGINE Bus STARTED 2024-04-02T04:48:58.920 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:58 smithi086 ceph-mon[29505]: mgrmap e18: y(active, since 1.97448s), standbys: x 2024-04-02T04:48:58.920 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:58 smithi086 ceph-mon[29505]: from='client.34316 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi086=a;smithi192=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:48:58.920 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:58 smithi086 ceph-mon[29505]: Saving service node-exporter spec with placement smithi086=a;smithi192=b;count:2 2024-04-02T04:48:58.920 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:58 smithi086 ceph-mon[29505]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:48:58.921 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:58 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:48:58.921 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:58 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:48:58.921 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:58 smithi086 ceph-mon[24811]: [02/Apr/2024:04:48:57] ENGINE Bus STARTING 2024-04-02T04:48:58.921 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:58 smithi086 ceph-mon[24811]: [02/Apr/2024:04:48:57] ENGINE Serving on https://172.21.15.86:7150 2024-04-02T04:48:58.921 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:58 smithi086 ceph-mon[24811]: [02/Apr/2024:04:48:57] ENGINE Bus STARTED 2024-04-02T04:48:58.921 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:58 smithi086 ceph-mon[24811]: mgrmap e18: y(active, since 1.97448s), standbys: x 2024-04-02T04:48:58.921 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:58 smithi086 ceph-mon[24811]: from='client.34316 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi086=a;smithi192=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:48:58.921 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:58 smithi086 ceph-mon[24811]: Saving service node-exporter spec with placement smithi086=a;smithi192=b;count:2 2024-04-02T04:48:58.921 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:58 smithi086 ceph-mon[24811]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:48:58.921 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:58 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:48:58.921 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:58 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:48:59.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:59 smithi192 ceph-mon[25697]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:48:59.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:59 smithi192 ceph-mon[25697]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-02T04:48:59.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:59 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:48:59.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:48:59 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:00.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:59 smithi086 ceph-mon[24811]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:00.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:59 smithi086 ceph-mon[24811]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-02T04:49:00.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:59 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:00.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:48:59 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:00.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:59 smithi086 ceph-mon[29505]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:00.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:59 smithi086 ceph-mon[29505]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-02T04:49:00.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:59 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:00.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:48:59 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:00.580 INFO:teuthology.orchestra.run.smithi192.stdout:Scheduled alertmanager update... 2024-04-02T04:49:00.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:00 smithi192 ceph-mon[25697]: Updating smithi086:/etc/ceph/ceph.conf 2024-04-02T04:49:00.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:00 smithi192 ceph-mon[25697]: Updating smithi086:/etc/ceph/ceph.client.admin.keyring 2024-04-02T04:49:00.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:00 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:00.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:00 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:01.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:00 smithi086 ceph-mon[29505]: Updating smithi086:/etc/ceph/ceph.conf 2024-04-02T04:49:01.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:00 smithi086 ceph-mon[29505]: Updating smithi086:/etc/ceph/ceph.client.admin.keyring 2024-04-02T04:49:01.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:00 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:01.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:00 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:01.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:00 smithi086 ceph-mon[24811]: Updating smithi086:/etc/ceph/ceph.conf 2024-04-02T04:49:01.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:00 smithi086 ceph-mon[24811]: Updating smithi086:/etc/ceph/ceph.client.admin.keyring 2024-04-02T04:49:01.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:00 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:01.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:00 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:01.120 DEBUG:teuthology.orchestra.run.smithi086:alertmanager.a> sudo journalctl -f -n 0 -u ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@alertmanager.a.service 2024-04-02T04:49:01.122 INFO:tasks.cephadm:Adding grafana.a on smithi192 2024-04-02T04:49:01.123 DEBUG:teuthology.orchestra.run.smithi192:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph orch apply grafana '1;smithi192=a' 2024-04-02T04:49:01.686 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:01 smithi192 ceph-mon[25697]: from='client.34337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi086=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:49:01.686 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:01 smithi192 ceph-mon[25697]: Saving service alertmanager spec with placement smithi086=a;count:1 2024-04-02T04:49:01.686 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:01 smithi192 ceph-mon[25697]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:01.686 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:01 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:02.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:01 smithi086 ceph-mon[29505]: from='client.34337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi086=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:49:02.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:01 smithi086 ceph-mon[29505]: Saving service alertmanager spec with placement smithi086=a;count:1 2024-04-02T04:49:02.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:01 smithi086 ceph-mon[29505]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:02.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:01 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:02.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:01 smithi086 ceph-mon[24811]: from='client.34337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi086=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:49:02.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:01 smithi086 ceph-mon[24811]: Saving service alertmanager spec with placement smithi086=a;count:1 2024-04-02T04:49:02.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:01 smithi086 ceph-mon[24811]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:02.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:01 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:02.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:02 smithi192 ceph-mon[25697]: Updating smithi192:/etc/ceph/ceph.conf 2024-04-02T04:49:02.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:02 smithi192 ceph-mon[25697]: Updating smithi192:/etc/ceph/ceph.client.admin.keyring 2024-04-02T04:49:02.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:02 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:02.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:02 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:03.008 INFO:teuthology.orchestra.run.smithi192.stdout:Scheduled grafana update... 2024-04-02T04:49:03.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:02 smithi086 ceph-mon[24811]: Updating smithi192:/etc/ceph/ceph.conf 2024-04-02T04:49:03.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:02 smithi086 ceph-mon[24811]: Updating smithi192:/etc/ceph/ceph.client.admin.keyring 2024-04-02T04:49:03.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:02 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:03.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:02 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:03.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:02 smithi086 ceph-mon[29505]: Updating smithi192:/etc/ceph/ceph.conf 2024-04-02T04:49:03.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:02 smithi086 ceph-mon[29505]: Updating smithi192:/etc/ceph/ceph.client.admin.keyring 2024-04-02T04:49:03.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:02 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:03.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:02 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:03.746 DEBUG:teuthology.orchestra.run.smithi192:grafana.a> sudo journalctl -f -n 0 -u ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@grafana.a.service 2024-04-02T04:49:03.748 INFO:tasks.cephadm:Setting up client nodes... 2024-04-02T04:49:03.751 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-02T04:49:03.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:03 smithi192 ceph-mon[25697]: Deploying daemon prometheus.a on smithi192 2024-04-02T04:49:03.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:03 smithi192 ceph-mon[25697]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:03.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:03 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:04.058 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:03 smithi086 ceph-mon[29505]: Deploying daemon prometheus.a on smithi192 2024-04-02T04:49:04.058 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:03 smithi086 ceph-mon[29505]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:04.058 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:03 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:04.058 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:03 smithi086 ceph-mon[24811]: Deploying daemon prometheus.a on smithi192 2024-04-02T04:49:04.059 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:03 smithi086 ceph-mon[24811]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:04.059 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:03 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:05.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:04 smithi086 ceph-mon[29505]: from='client.34343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi192=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:49:05.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:04 smithi086 ceph-mon[29505]: Saving service grafana spec with placement smithi192=a;count:1 2024-04-02T04:49:05.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:04 smithi086 ceph-mon[24811]: from='client.34343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi192=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:49:05.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:04 smithi086 ceph-mon[24811]: Saving service grafana spec with placement smithi192=a;count:1 2024-04-02T04:49:05.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:04 smithi192 ceph-mon[25697]: from='client.34343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi192=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:49:05.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:04 smithi192 ceph-mon[25697]: Saving service grafana spec with placement smithi192=a;count:1 2024-04-02T04:49:05.760 INFO:teuthology.orchestra.run.smithi086.stdout:[client.0] 2024-04-02T04:49:05.760 INFO:teuthology.orchestra.run.smithi086.stdout: key = AQBBjgtmURT5LBAA9/YpkzUCUY9N0R/LBXDPXw== 2024-04-02T04:49:06.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:05 smithi086 ceph-mon[29505]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:06.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:05 smithi086 ceph-mon[24811]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:06.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:05 smithi192 ceph-mon[25697]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:06.395 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-04-02T04:49:06.395 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-04-02T04:49:06.395 DEBUG:teuthology.orchestra.run.smithi086:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-04-02T04:49:06.438 DEBUG:teuthology.orchestra.run.smithi192:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-02T04:49:06.742 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:06 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/3493057244' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T04:49:06.743 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:06 smithi192 ceph-mon[25697]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T04:49:06.743 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:06 smithi192 ceph-mon[25697]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-02T04:49:07.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:06 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/3493057244' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T04:49:07.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:06 smithi086 ceph-mon[29505]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T04:49:07.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:06 smithi086 ceph-mon[29505]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-02T04:49:07.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:06 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/3493057244' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T04:49:07.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:06 smithi086 ceph-mon[24811]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T04:49:07.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:06 smithi086 ceph-mon[24811]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-02T04:49:08.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:07 smithi086 ceph-mon[24811]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:08.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:07 smithi086 ceph-mon[29505]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:08.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:07 smithi192 ceph-mon[25697]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:10.013 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:09 smithi192 ceph-mon[25697]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:10.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:09 smithi086 ceph-mon[24811]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:10.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:09 smithi086 ceph-mon[29505]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:10.222 INFO:teuthology.orchestra.run.smithi192.stdout:[client.1] 2024-04-02T04:49:10.222 INFO:teuthology.orchestra.run.smithi192.stdout: key = AQBGjgtmzYm2DBAAqe1SPbdhUwewSp34eq1VOg== 2024-04-02T04:49:10.824 DEBUG:teuthology.orchestra.run.smithi192:> set -ex 2024-04-02T04:49:10.824 DEBUG:teuthology.orchestra.run.smithi192:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-04-02T04:49:10.824 DEBUG:teuthology.orchestra.run.smithi192:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-04-02T04:49:10.866 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-02T04:49:10.866 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-02T04:49:10.867 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph mgr dump --format=json 2024-04-02T04:49:11.070 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:10 smithi192 ceph-mon[25697]: from='client.? 172.21.15.192:0/2968114768' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T04:49:11.070 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:10 smithi192 ceph-mon[25697]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T04:49:11.070 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:10 smithi192 ceph-mon[25697]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-02T04:49:11.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:10 smithi086 ceph-mon[24811]: from='client.? 172.21.15.192:0/2968114768' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T04:49:11.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:10 smithi086 ceph-mon[24811]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T04:49:11.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:10 smithi086 ceph-mon[24811]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-02T04:49:11.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:10 smithi086 ceph-mon[29505]: from='client.? 172.21.15.192:0/2968114768' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T04:49:11.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:10 smithi086 ceph-mon[29505]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T04:49:11.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:10 smithi086 ceph-mon[29505]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-02T04:49:11.086 INFO:teuthology.orchestra.run.smithi086.stderr:Inferring config /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T04:49:12.042 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:11 smithi192 ceph-mon[25697]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:12.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:11 smithi086 ceph-mon[24811]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:12.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:11 smithi086 ceph-mon[29505]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:12.809 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-02T04:49:13.449 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":19,"active_gid":14424,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":1542923653},{"type":"v1","addr":"172.21.15.86:6801","nonce":1542923653}]},"active_addr":"172.21.15.86:6801/1542923653","active_change":"2024-04-02T04:48:55.682970+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":34307,"name":"x","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.86:8443/","prometheus":"http://172.21.15.86:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":59,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.86:0","nonce":3680953997}]},{"addrvec":[{"type":"v2","addr":"172.21.15.86:0","nonce":1206491125}]},{"addrvec":[{"type":"v2","addr":"172.21.15.86:0","nonce":2034239143}]},{"addrvec":[{"type":"v2","addr":"172.21.15.86:0","nonce":1119175753}]}]}} 2024-04-02T04:49:13.452 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-02T04:49:13.452 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-02T04:49:13.453 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-02T04:49:13.668 INFO:teuthology.orchestra.run.smithi086.stderr:Inferring config /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T04:49:14.023 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:13 smithi086 ceph-mon[24811]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:14.023 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:13 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/2066077240' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-02T04:49:14.024 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:13 smithi086 ceph-mon[29505]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:14.024 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:13 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/2066077240' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-02T04:49:14.079 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:13 smithi192 ceph-mon[25697]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:14.079 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:13 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/2066077240' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-02T04:49:14.080 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:49:13 smithi192 systemd[1]: Starting Ceph prometheus.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T04:49:14.339 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:49:14 smithi192 podman[40802]: 2024-04-02 04:49:13.980554375 +0000 UTC m=+0.018566219 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-02T04:49:14.339 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:49:14 smithi192 podman[40802]: 2024-04-02 04:49:14.091775117 +0000 UTC m=+0.129786992 container create 93d642a7521e1350074c2ee753bfddffcb1b9f5299b96b339667bc83a0d84f6f (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T04:49:14.339 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:49:14 smithi192 podman[40802]: 2024-04-02 04:49:14.310844981 +0000 UTC m=+0.348856821 container init 93d642a7521e1350074c2ee753bfddffcb1b9f5299b96b339667bc83a0d84f6f (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T04:49:14.340 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:49:14 smithi192 podman[40802]: 2024-04-02 04:49:14.313464171 +0000 UTC m=+0.351476008 container start 93d642a7521e1350074c2ee753bfddffcb1b9f5299b96b339667bc83a0d84f6f (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T04:49:14.678 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:49:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[40817]: ts=2024-04-02T04:49:14.336Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-02T04:49:14.679 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:49:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[40817]: ts=2024-04-02T04:49:14.336Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-02T04:49:14.679 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:49:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[40817]: ts=2024-04-02T04:49:14.337Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-02T04:49:14.679 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:49:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[40817]: ts=2024-04-02T04:49:14.337Z caller=main.go:518 level=info host_details="(Linux 5.14.0-434.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Mar 26 19:20:47 UTC 2024 x86_64 smithi192 (none))" 2024-04-02T04:49:14.679 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:49:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[40817]: ts=2024-04-02T04:49:14.337Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-02T04:49:14.679 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:49:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[40817]: ts=2024-04-02T04:49:14.337Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-02T04:49:14.679 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:49:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[40817]: ts=2024-04-02T04:49:14.338Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-02T04:49:14.679 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:49:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[40817]: ts=2024-04-02T04:49:14.339Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-02T04:49:14.679 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:49:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[40817]: ts=2024-04-02T04:49:14.339Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-02T04:49:14.679 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:49:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[40817]: ts=2024-04-02T04:49:14.341Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-02T04:49:14.679 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:49:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[40817]: ts=2024-04-02T04:49:14.341Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.231µs 2024-04-02T04:49:14.679 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:49:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[40817]: ts=2024-04-02T04:49:14.341Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-02T04:49:14.679 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:49:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[40817]: ts=2024-04-02T04:49:14.341Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-04-02T04:49:14.679 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:49:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[40817]: ts=2024-04-02T04:49:14.341Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=28.975µs wal_replay_duration=105.288µs total_replay_duration=156.419µs 2024-04-02T04:49:14.680 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:49:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[40817]: ts=2024-04-02T04:49:14.341Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-02T04:49:14.680 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:49:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[40817]: ts=2024-04-02T04:49:14.341Z caller=main.go:947 level=info msg="TSDB started" 2024-04-02T04:49:14.680 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:49:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[40817]: ts=2024-04-02T04:49:14.341Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-02T04:49:14.680 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:49:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[40817]: ts=2024-04-02T04:49:14.357Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.300259ms db_storage=643ns remote_storage=1.172µs web_handler=305ns query_engine=574ns scrape=766.009µs scrape_sd=54.455µs notify=1.14µs notify_sd=2.512µs rules=14.306799ms 2024-04-02T04:49:14.680 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:49:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[40817]: ts=2024-04-02T04:49:14.357Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-02T04:49:14.680 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:49:14 smithi192 bash[40802]: 93d642a7521e1350074c2ee753bfddffcb1b9f5299b96b339667bc83a0d84f6f 2024-04-02T04:49:14.680 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:49:14 smithi192 systemd[1]: Started Ceph prometheus.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T04:49:15.321 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-02T04:49:15.321 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":59,"fsid":"8862e8fe-f0ab-11ee-b647-cb9ed24678a4","created":"2024-04-02T04:43:51.159039+0000","modified":"2024-04-02T04:48:55.682590+0000","last_up_change":"2024-04-02T04:48:40.627055+0000","last_in_change":"2024-04-02T04:48:24.210591+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-02T04:46:48.351740+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"e6ee7c83-68a3-4d34-8cdc-21972cf722bf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":2621947859},{"type":"v1","addr":"172.21.15.86:6803","nonce":2621947859}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":2621947859},{"type":"v1","addr":"172.21.15.86:6805","nonce":2621947859}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2621947859},{"type":"v1","addr":"172.21.15.86:6809","nonce":2621947859}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":2621947859},{"type":"v1","addr":"172.21.15.86:6807","nonce":2621947859}]},"public_addr":"172.21.15.86:6803/2621947859","cluster_addr":"172.21.15.86:6805/2621947859","heartbeat_back_addr":"172.21.15.86:6809/2621947859","heartbeat_front_addr":"172.21.15.86:6807/2621947859","state":["exists","up"]},{"osd":1,"uuid":"fb117c77-5aff-44fc-bcfd-23805e94220d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6810","nonce":3495515130},{"type":"v1","addr":"172.21.15.86:6811","nonce":3495515130}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":3495515130},{"type":"v1","addr":"172.21.15.86:6813","nonce":3495515130}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":3495515130},{"type":"v1","addr":"172.21.15.86:6817","nonce":3495515130}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6814","nonce":3495515130},{"type":"v1","addr":"172.21.15.86:6815","nonce":3495515130}]},"public_addr":"172.21.15.86:6811/3495515130","cluster_addr":"172.21.15.86:6813/3495515130","heartbeat_back_addr":"172.21.15.86:6817/3495515130","heartbeat_front_addr":"172.21.15.86:6815/3495515130","state":["exists","up"]},{"osd":2,"uuid":"a0ee7998-0bb3-4b9d-b536-d305b6679aa6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6818","nonce":1835017224},{"type":"v1","addr":"172.21.15.86:6819","nonce":1835017224}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":1835017224},{"type":"v1","addr":"172.21.15.86:6821","nonce":1835017224}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":1835017224},{"type":"v1","addr":"172.21.15.86:6825","nonce":1835017224}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6822","nonce":1835017224},{"type":"v1","addr":"172.21.15.86:6823","nonce":1835017224}]},"public_addr":"172.21.15.86:6819/1835017224","cluster_addr":"172.21.15.86:6821/1835017224","heartbeat_back_addr":"172.21.15.86:6825/1835017224","heartbeat_front_addr":"172.21.15.86:6823/1835017224","state":["exists","up"]},{"osd":3,"uuid":"58bbbf2d-749e-4534-8800-0e945099e4bc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":1252559528},{"type":"v1","addr":"172.21.15.86:6827","nonce":1252559528}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":1252559528},{"type":"v1","addr":"172.21.15.86:6829","nonce":1252559528}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6832","nonce":1252559528},{"type":"v1","addr":"172.21.15.86:6833","nonce":1252559528}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":1252559528},{"type":"v1","addr":"172.21.15.86:6831","nonce":1252559528}]},"public_addr":"172.21.15.86:6827/1252559528","cluster_addr":"172.21.15.86:6829/1252559528","heartbeat_back_addr":"172.21.15.86:6833/1252559528","heartbeat_front_addr":"172.21.15.86:6831/1252559528","state":["exists","up"]},{"osd":4,"uuid":"b0021f30-c92c-43dc-898e-99dd4618ef4e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6800","nonce":2803467108},{"type":"v1","addr":"172.21.15.192:6801","nonce":2803467108}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6802","nonce":2803467108},{"type":"v1","addr":"172.21.15.192:6803","nonce":2803467108}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6806","nonce":2803467108},{"type":"v1","addr":"172.21.15.192:6807","nonce":2803467108}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6804","nonce":2803467108},{"type":"v1","addr":"172.21.15.192:6805","nonce":2803467108}]},"public_addr":"172.21.15.192:6801/2803467108","cluster_addr":"172.21.15.192:6803/2803467108","heartbeat_back_addr":"172.21.15.192:6807/2803467108","heartbeat_front_addr":"172.21.15.192:6805/2803467108","state":["exists","up"]},{"osd":5,"uuid":"a3246bff-975b-40b4-90fc-daa551484ec9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6808","nonce":1908092348},{"type":"v1","addr":"172.21.15.192:6809","nonce":1908092348}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6810","nonce":1908092348},{"type":"v1","addr":"172.21.15.192:6811","nonce":1908092348}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6814","nonce":1908092348},{"type":"v1","addr":"172.21.15.192:6815","nonce":1908092348}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6812","nonce":1908092348},{"type":"v1","addr":"172.21.15.192:6813","nonce":1908092348}]},"public_addr":"172.21.15.192:6809/1908092348","cluster_addr":"172.21.15.192:6811/1908092348","heartbeat_back_addr":"172.21.15.192:6815/1908092348","heartbeat_front_addr":"172.21.15.192:6813/1908092348","state":["exists","up"]},{"osd":6,"uuid":"b4eab3ce-dfc9-41d9-95bd-98135ed757a3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6816","nonce":2019998998},{"type":"v1","addr":"172.21.15.192:6817","nonce":2019998998}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6818","nonce":2019998998},{"type":"v1","addr":"172.21.15.192:6819","nonce":2019998998}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6822","nonce":2019998998},{"type":"v1","addr":"172.21.15.192:6823","nonce":2019998998}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6820","nonce":2019998998},{"type":"v1","addr":"172.21.15.192:6821","nonce":2019998998}]},"public_addr":"172.21.15.192:6817/2019998998","cluster_addr":"172.21.15.192:6819/2019998998","heartbeat_back_addr":"172.21.15.192:6823/2019998998","heartbeat_front_addr":"172.21.15.192:6821/2019998998","state":["exists","up"]},{"osd":7,"uuid":"f1ddd0fd-cf2e-4abf-a682-9c686712b926","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6824","nonce":2081381372},{"type":"v1","addr":"172.21.15.192:6825","nonce":2081381372}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6826","nonce":2081381372},{"type":"v1","addr":"172.21.15.192:6827","nonce":2081381372}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6830","nonce":2081381372},{"type":"v1","addr":"172.21.15.192:6831","nonce":2081381372}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6828","nonce":2081381372},{"type":"v1","addr":"172.21.15.192:6829","nonce":2081381372}]},"public_addr":"172.21.15.192:6825/2081381372","cluster_addr":"172.21.15.192:6827/2081381372","heartbeat_back_addr":"172.21.15.192:6831/2081381372","heartbeat_front_addr":"172.21.15.192:6829/2081381372","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:46:00.645602+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:46:22.956048+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:46:45.818580+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:47:09.397500+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:47:30.864808+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:47:53.134635+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:48:16.072372+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:48:39.225607+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.86:0/368218600":"2024-04-03T04:48:55.682550+0000","172.21.15.86:0/594554291":"2024-04-03T04:48:55.682550+0000","172.21.15.86:6801/939688624":"2024-04-03T04:48:55.682550+0000","172.21.15.86:0/1928070731":"2024-04-03T04:48:55.682550+0000","172.21.15.86:0/2885713909":"2024-04-03T04:44:43.987095+0000","172.21.15.86:0/3311785205":"2024-04-03T04:48:55.682550+0000","172.21.15.86:0/2853453946":"2024-04-03T04:44:43.987095+0000","172.21.15.86:6801/3904718154":"2024-04-03T04:44:43.987095+0000","172.21.15.86:6800/3904718154":"2024-04-03T04:44:43.987095+0000","172.21.15.86:0/4053196327":"2024-04-03T04:44:22.479015+0000","172.21.15.86:0/1081336665":"2024-04-03T04:44:22.479015+0000","172.21.15.86:6800/1417258151":"2024-04-03T04:44:22.479015+0000","172.21.15.86:6800/939688624":"2024-04-03T04:48:55.682550+0000","172.21.15.86:0/2634385848":"2024-04-03T04:44:43.987095+0000","172.21.15.86:0/1565660562":"2024-04-03T04:44:22.479015+0000","172.21.15.86:6801/1417258151":"2024-04-03T04:44:22.479015+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-02T04:49:15.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:15 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:15.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:15 smithi086 ceph-mon[29505]: Deploying daemon node-exporter.a on smithi086 2024-04-02T04:49:15.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:15 smithi086 ceph-mon[29505]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:15.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:15 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1082562346' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T04:49:15.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:15 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:15.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:15 smithi086 ceph-mon[24811]: Deploying daemon node-exporter.a on smithi086 2024-04-02T04:49:15.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:15 smithi086 ceph-mon[24811]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:15.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:15 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1082562346' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T04:49:15.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:15 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:15.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:15 smithi192 ceph-mon[25697]: Deploying daemon node-exporter.a on smithi086 2024-04-02T04:49:15.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:15 smithi192 ceph-mon[25697]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:15.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:15 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1082562346' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T04:49:16.023 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-02T04:49:16.023 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-02T04:49:16.260 INFO:teuthology.orchestra.run.smithi086.stderr:Inferring config /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T04:49:16.333 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:16 smithi086 bash[44430]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2024-04-02T04:49:16.737 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:16 smithi086 bash[44430]: Getting image source signatures 2024-04-02T04:49:16.737 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:16 smithi086 bash[44430]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2024-04-02T04:49:16.737 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:16 smithi086 bash[44430]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2024-04-02T04:49:16.737 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:16 smithi086 bash[44430]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-04-02T04:49:17.971 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:17 smithi086 ceph-mon[24811]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:17.971 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:17 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:17.971 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:17 smithi086 ceph-mon[29505]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:17.972 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:17 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:18.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:17 smithi192 ceph-mon[25697]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:18.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:17 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:18.665 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:18 smithi086 bash[44430]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-04-02T04:49:18.666 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:18 smithi086 bash[44430]: Writing manifest to image destination 2024-04-02T04:49:19.031 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-02T04:49:19.031 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":59,"fsid":"8862e8fe-f0ab-11ee-b647-cb9ed24678a4","created":"2024-04-02T04:43:51.159039+0000","modified":"2024-04-02T04:48:55.682590+0000","last_up_change":"2024-04-02T04:48:40.627055+0000","last_in_change":"2024-04-02T04:48:24.210591+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-02T04:46:48.351740+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"e6ee7c83-68a3-4d34-8cdc-21972cf722bf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":2621947859},{"type":"v1","addr":"172.21.15.86:6803","nonce":2621947859}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":2621947859},{"type":"v1","addr":"172.21.15.86:6805","nonce":2621947859}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":2621947859},{"type":"v1","addr":"172.21.15.86:6809","nonce":2621947859}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":2621947859},{"type":"v1","addr":"172.21.15.86:6807","nonce":2621947859}]},"public_addr":"172.21.15.86:6803/2621947859","cluster_addr":"172.21.15.86:6805/2621947859","heartbeat_back_addr":"172.21.15.86:6809/2621947859","heartbeat_front_addr":"172.21.15.86:6807/2621947859","state":["exists","up"]},{"osd":1,"uuid":"fb117c77-5aff-44fc-bcfd-23805e94220d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6810","nonce":3495515130},{"type":"v1","addr":"172.21.15.86:6811","nonce":3495515130}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":3495515130},{"type":"v1","addr":"172.21.15.86:6813","nonce":3495515130}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":3495515130},{"type":"v1","addr":"172.21.15.86:6817","nonce":3495515130}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6814","nonce":3495515130},{"type":"v1","addr":"172.21.15.86:6815","nonce":3495515130}]},"public_addr":"172.21.15.86:6811/3495515130","cluster_addr":"172.21.15.86:6813/3495515130","heartbeat_back_addr":"172.21.15.86:6817/3495515130","heartbeat_front_addr":"172.21.15.86:6815/3495515130","state":["exists","up"]},{"osd":2,"uuid":"a0ee7998-0bb3-4b9d-b536-d305b6679aa6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6818","nonce":1835017224},{"type":"v1","addr":"172.21.15.86:6819","nonce":1835017224}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":1835017224},{"type":"v1","addr":"172.21.15.86:6821","nonce":1835017224}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":1835017224},{"type":"v1","addr":"172.21.15.86:6825","nonce":1835017224}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6822","nonce":1835017224},{"type":"v1","addr":"172.21.15.86:6823","nonce":1835017224}]},"public_addr":"172.21.15.86:6819/1835017224","cluster_addr":"172.21.15.86:6821/1835017224","heartbeat_back_addr":"172.21.15.86:6825/1835017224","heartbeat_front_addr":"172.21.15.86:6823/1835017224","state":["exists","up"]},{"osd":3,"uuid":"58bbbf2d-749e-4534-8800-0e945099e4bc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":1252559528},{"type":"v1","addr":"172.21.15.86:6827","nonce":1252559528}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":1252559528},{"type":"v1","addr":"172.21.15.86:6829","nonce":1252559528}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6832","nonce":1252559528},{"type":"v1","addr":"172.21.15.86:6833","nonce":1252559528}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":1252559528},{"type":"v1","addr":"172.21.15.86:6831","nonce":1252559528}]},"public_addr":"172.21.15.86:6827/1252559528","cluster_addr":"172.21.15.86:6829/1252559528","heartbeat_back_addr":"172.21.15.86:6833/1252559528","heartbeat_front_addr":"172.21.15.86:6831/1252559528","state":["exists","up"]},{"osd":4,"uuid":"b0021f30-c92c-43dc-898e-99dd4618ef4e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6800","nonce":2803467108},{"type":"v1","addr":"172.21.15.192:6801","nonce":2803467108}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6802","nonce":2803467108},{"type":"v1","addr":"172.21.15.192:6803","nonce":2803467108}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6806","nonce":2803467108},{"type":"v1","addr":"172.21.15.192:6807","nonce":2803467108}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6804","nonce":2803467108},{"type":"v1","addr":"172.21.15.192:6805","nonce":2803467108}]},"public_addr":"172.21.15.192:6801/2803467108","cluster_addr":"172.21.15.192:6803/2803467108","heartbeat_back_addr":"172.21.15.192:6807/2803467108","heartbeat_front_addr":"172.21.15.192:6805/2803467108","state":["exists","up"]},{"osd":5,"uuid":"a3246bff-975b-40b4-90fc-daa551484ec9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6808","nonce":1908092348},{"type":"v1","addr":"172.21.15.192:6809","nonce":1908092348}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6810","nonce":1908092348},{"type":"v1","addr":"172.21.15.192:6811","nonce":1908092348}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6814","nonce":1908092348},{"type":"v1","addr":"172.21.15.192:6815","nonce":1908092348}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6812","nonce":1908092348},{"type":"v1","addr":"172.21.15.192:6813","nonce":1908092348}]},"public_addr":"172.21.15.192:6809/1908092348","cluster_addr":"172.21.15.192:6811/1908092348","heartbeat_back_addr":"172.21.15.192:6815/1908092348","heartbeat_front_addr":"172.21.15.192:6813/1908092348","state":["exists","up"]},{"osd":6,"uuid":"b4eab3ce-dfc9-41d9-95bd-98135ed757a3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6816","nonce":2019998998},{"type":"v1","addr":"172.21.15.192:6817","nonce":2019998998}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6818","nonce":2019998998},{"type":"v1","addr":"172.21.15.192:6819","nonce":2019998998}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6822","nonce":2019998998},{"type":"v1","addr":"172.21.15.192:6823","nonce":2019998998}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6820","nonce":2019998998},{"type":"v1","addr":"172.21.15.192:6821","nonce":2019998998}]},"public_addr":"172.21.15.192:6817/2019998998","cluster_addr":"172.21.15.192:6819/2019998998","heartbeat_back_addr":"172.21.15.192:6823/2019998998","heartbeat_front_addr":"172.21.15.192:6821/2019998998","state":["exists","up"]},{"osd":7,"uuid":"f1ddd0fd-cf2e-4abf-a682-9c686712b926","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6824","nonce":2081381372},{"type":"v1","addr":"172.21.15.192:6825","nonce":2081381372}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6826","nonce":2081381372},{"type":"v1","addr":"172.21.15.192:6827","nonce":2081381372}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6830","nonce":2081381372},{"type":"v1","addr":"172.21.15.192:6831","nonce":2081381372}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.192:6828","nonce":2081381372},{"type":"v1","addr":"172.21.15.192:6829","nonce":2081381372}]},"public_addr":"172.21.15.192:6825/2081381372","cluster_addr":"172.21.15.192:6827/2081381372","heartbeat_back_addr":"172.21.15.192:6831/2081381372","heartbeat_front_addr":"172.21.15.192:6829/2081381372","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:46:00.645602+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:46:22.956048+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:46:45.818580+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:47:09.397500+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:47:30.864808+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:47:53.134635+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:48:16.072372+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:48:39.225607+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.86:0/368218600":"2024-04-03T04:48:55.682550+0000","172.21.15.86:0/594554291":"2024-04-03T04:48:55.682550+0000","172.21.15.86:6801/939688624":"2024-04-03T04:48:55.682550+0000","172.21.15.86:0/1928070731":"2024-04-03T04:48:55.682550+0000","172.21.15.86:0/2885713909":"2024-04-03T04:44:43.987095+0000","172.21.15.86:0/3311785205":"2024-04-03T04:48:55.682550+0000","172.21.15.86:0/2853453946":"2024-04-03T04:44:43.987095+0000","172.21.15.86:6801/3904718154":"2024-04-03T04:44:43.987095+0000","172.21.15.86:6800/3904718154":"2024-04-03T04:44:43.987095+0000","172.21.15.86:0/4053196327":"2024-04-03T04:44:22.479015+0000","172.21.15.86:0/1081336665":"2024-04-03T04:44:22.479015+0000","172.21.15.86:6800/1417258151":"2024-04-03T04:44:22.479015+0000","172.21.15.86:6800/939688624":"2024-04-03T04:48:55.682550+0000","172.21.15.86:0/2634385848":"2024-04-03T04:44:43.987095+0000","172.21.15.86:0/1565660562":"2024-04-03T04:44:22.479015+0000","172.21.15.86:6801/1417258151":"2024-04-03T04:44:22.479015+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-02T04:49:19.333 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:19 smithi086 podman[44430]: 2024-04-02 04:49:19.111044104 +0000 UTC m=+2.998179450 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-04-02T04:49:19.621 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph tell osd.0 flush_pg_stats 2024-04-02T04:49:19.622 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph tell osd.1 flush_pg_stats 2024-04-02T04:49:19.622 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph tell osd.2 flush_pg_stats 2024-04-02T04:49:19.622 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph tell osd.3 flush_pg_stats 2024-04-02T04:49:19.623 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph tell osd.4 flush_pg_stats 2024-04-02T04:49:19.623 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph tell osd.5 flush_pg_stats 2024-04-02T04:49:19.624 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph tell osd.6 flush_pg_stats 2024-04-02T04:49:19.624 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph tell osd.7 flush_pg_stats 2024-04-02T04:49:19.917 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:19 smithi086 ceph-mon[24811]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:19.917 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:19 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/749966127' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T04:49:19.918 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:19 smithi086 ceph-mon[29505]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:19.918 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:19 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/749966127' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T04:49:19.919 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:19 smithi086 podman[44430]: 2024-04-02 04:49:19.804741865 +0000 UTC m=+3.691877206 container create 0151f4e72bd9ab8de43694ed7be25b1a230c9320106f0c2b09ce93fde02a7d94 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-02T04:49:20.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:19 smithi192 ceph-mon[25697]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:20.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:19 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/749966127' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T04:49:20.237 INFO:teuthology.orchestra.run.smithi086.stderr:Inferring config /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T04:49:20.237 INFO:teuthology.orchestra.run.smithi086.stderr:Inferring config /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T04:49:20.237 INFO:teuthology.orchestra.run.smithi086.stderr:Inferring config /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T04:49:20.333 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:49:20] "GET /metrics HTTP/1.1" 200 191497 "" "Prometheus/2.33.4" 2024-04-02T04:49:20.333 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 podman[44430]: 2024-04-02 04:49:20.040729922 +0000 UTC m=+3.927865254 container init 0151f4e72bd9ab8de43694ed7be25b1a230c9320106f0c2b09ce93fde02a7d94 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-02T04:49:20.334 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 podman[44430]: 2024-04-02 04:49:20.043480463 +0000 UTC m=+3.930615809 container start 0151f4e72bd9ab8de43694ed7be25b1a230c9320106f0c2b09ce93fde02a7d94 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-02T04:49:20.334 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.047Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-04-02T04:49:20.334 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.047Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-04-02T04:49:20.334 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.047Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-04-02T04:49:20.334 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.047Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-02T04:49:20.334 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-04-02T04:49:20.334 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=arp 2024-04-02T04:49:20.334 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=bcache 2024-04-02T04:49:20.334 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=bonding 2024-04-02T04:49:20.334 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=btrfs 2024-04-02T04:49:20.334 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=conntrack 2024-04-02T04:49:20.334 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=cpu 2024-04-02T04:49:20.334 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-04-02T04:49:20.334 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=diskstats 2024-04-02T04:49:20.335 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=dmi 2024-04-02T04:49:20.335 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=edac 2024-04-02T04:49:20.335 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=entropy 2024-04-02T04:49:20.335 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-04-02T04:49:20.335 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=filefd 2024-04-02T04:49:20.335 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=filesystem 2024-04-02T04:49:20.335 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=hwmon 2024-04-02T04:49:20.336 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=infiniband 2024-04-02T04:49:20.336 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=ipvs 2024-04-02T04:49:20.336 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=loadavg 2024-04-02T04:49:20.336 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=mdadm 2024-04-02T04:49:20.336 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=meminfo 2024-04-02T04:49:20.336 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=netclass 2024-04-02T04:49:20.336 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=netdev 2024-04-02T04:49:20.336 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=netstat 2024-04-02T04:49:20.336 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=nfs 2024-04-02T04:49:20.336 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=nfsd 2024-04-02T04:49:20.336 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=nvme 2024-04-02T04:49:20.336 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=os 2024-04-02T04:49:20.336 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-04-02T04:49:20.336 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=pressure 2024-04-02T04:49:20.336 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=rapl 2024-04-02T04:49:20.337 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=schedstat 2024-04-02T04:49:20.337 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=sockstat 2024-04-02T04:49:20.337 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=softnet 2024-04-02T04:49:20.337 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=stat 2024-04-02T04:49:20.337 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=tapestats 2024-04-02T04:49:20.337 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=textfile 2024-04-02T04:49:20.337 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-04-02T04:49:20.337 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=time 2024-04-02T04:49:20.337 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-04-02T04:49:20.337 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=uname 2024-04-02T04:49:20.337 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=vmstat 2024-04-02T04:49:20.337 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=xfs 2024-04-02T04:49:20.337 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:115 level=info collector=zfs 2024-04-02T04:49:20.337 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-04-02T04:49:20.337 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[44754]: ts=2024-04-02T04:49:20.048Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-04-02T04:49:20.338 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 bash[44430]: 0151f4e72bd9ab8de43694ed7be25b1a230c9320106f0c2b09ce93fde02a7d94 2024-04-02T04:49:20.338 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:49:20 smithi086 systemd[1]: Started Ceph node-exporter.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T04:49:20.896 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:20 smithi192 systemd[1]: Starting Ceph node-exporter.b for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T04:49:21.178 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:21 smithi192 bash[41047]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2024-04-02T04:49:21.519 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:21 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:21.519 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:21 smithi086 ceph-mon[29505]: Deploying daemon node-exporter.b on smithi192 2024-04-02T04:49:21.519 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:21 smithi086 ceph-mon[29505]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:21.519 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:21 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:21.519 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:21 smithi086 ceph-mon[24811]: Deploying daemon node-exporter.b on smithi192 2024-04-02T04:49:21.519 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:21 smithi086 ceph-mon[24811]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:21.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:21 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:21.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:21 smithi192 ceph-mon[25697]: Deploying daemon node-exporter.b on smithi192 2024-04-02T04:49:21.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:21 smithi192 ceph-mon[25697]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:21.678 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:21 smithi192 bash[41047]: Getting image source signatures 2024-04-02T04:49:21.678 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:21 smithi192 bash[41047]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2024-04-02T04:49:21.678 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:21 smithi192 bash[41047]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-04-02T04:49:21.678 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:21 smithi192 bash[41047]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2024-04-02T04:49:23.038 INFO:teuthology.orchestra.run.smithi086.stderr:Inferring config /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T04:49:23.039 INFO:teuthology.orchestra.run.smithi086.stderr:Inferring config /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T04:49:23.177 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:22 smithi192 bash[41047]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-04-02T04:49:23.178 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 bash[41047]: Writing manifest to image destination 2024-04-02T04:49:23.467 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 podman[41047]: 2024-04-02 04:49:23.272932004 +0000 UTC m=+2.280649289 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-04-02T04:49:23.468 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 podman[41047]: 2024-04-02 04:49:23.371580525 +0000 UTC m=+2.379297802 container create 27476f65f84fb08b5794840b29ebfe4e358583fd9ae5edde0e62121aef6240d1 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-02T04:49:23.562 INFO:teuthology.orchestra.run.smithi086.stderr:Inferring config /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T04:49:23.562 INFO:teuthology.orchestra.run.smithi086.stderr:Inferring config /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T04:49:23.773 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:23 smithi086 ceph-mon[24811]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:23.773 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:23 smithi086 ceph-mon[29505]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:23.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-mon[25697]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:23.928 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 podman[41047]: 2024-04-02 04:49:23.573588716 +0000 UTC m=+2.581305991 container init 27476f65f84fb08b5794840b29ebfe4e358583fd9ae5edde0e62121aef6240d1 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-02T04:49:23.928 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 podman[41047]: 2024-04-02 04:49:23.576138004 +0000 UTC m=+2.583855278 container start 27476f65f84fb08b5794840b29ebfe4e358583fd9ae5edde0e62121aef6240d1 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-02T04:49:23.928 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.579Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-04-02T04:49:23.928 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.580Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-04-02T04:49:23.928 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.580Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-04-02T04:49:23.928 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.580Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-02T04:49:23.928 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-04-02T04:49:23.928 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=arp 2024-04-02T04:49:23.928 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=bcache 2024-04-02T04:49:23.928 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=bonding 2024-04-02T04:49:23.928 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=btrfs 2024-04-02T04:49:23.929 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=conntrack 2024-04-02T04:49:23.929 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=cpu 2024-04-02T04:49:23.929 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-04-02T04:49:23.929 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=diskstats 2024-04-02T04:49:23.929 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=dmi 2024-04-02T04:49:23.929 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=edac 2024-04-02T04:49:23.929 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=entropy 2024-04-02T04:49:23.929 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-04-02T04:49:23.930 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=filefd 2024-04-02T04:49:23.930 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=filesystem 2024-04-02T04:49:23.930 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=hwmon 2024-04-02T04:49:23.930 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=infiniband 2024-04-02T04:49:23.930 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=ipvs 2024-04-02T04:49:23.930 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=loadavg 2024-04-02T04:49:23.930 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=mdadm 2024-04-02T04:49:23.930 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=meminfo 2024-04-02T04:49:23.930 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=netclass 2024-04-02T04:49:23.930 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=netdev 2024-04-02T04:49:23.930 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=netstat 2024-04-02T04:49:23.930 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=nfs 2024-04-02T04:49:23.931 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=nfsd 2024-04-02T04:49:23.931 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=nvme 2024-04-02T04:49:23.931 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=os 2024-04-02T04:49:23.931 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-04-02T04:49:23.931 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=pressure 2024-04-02T04:49:23.931 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=rapl 2024-04-02T04:49:23.931 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=schedstat 2024-04-02T04:49:23.931 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=sockstat 2024-04-02T04:49:23.931 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=softnet 2024-04-02T04:49:23.931 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=stat 2024-04-02T04:49:23.931 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=tapestats 2024-04-02T04:49:23.931 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=textfile 2024-04-02T04:49:23.931 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-04-02T04:49:23.931 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=time 2024-04-02T04:49:23.931 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-04-02T04:49:23.932 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=uname 2024-04-02T04:49:23.932 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=vmstat 2024-04-02T04:49:23.932 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=xfs 2024-04-02T04:49:23.932 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:115 level=info collector=zfs 2024-04-02T04:49:23.932 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-04-02T04:49:23.932 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[41098]: ts=2024-04-02T04:49:23.581Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-04-02T04:49:23.932 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 bash[41047]: 27476f65f84fb08b5794840b29ebfe4e358583fd9ae5edde0e62121aef6240d1 2024-04-02T04:49:23.932 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:49:23 smithi192 systemd[1]: Started Ceph node-exporter.b for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T04:49:24.982 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:24 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:24.982 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:24 smithi086 ceph-mon[24811]: Deploying daemon alertmanager.a on smithi086 2024-04-02T04:49:24.982 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:24 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:24.982 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:24 smithi086 ceph-mon[29505]: Deploying daemon alertmanager.a on smithi086 2024-04-02T04:49:25.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:24 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:25.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:24 smithi192 ceph-mon[25697]: Deploying daemon alertmanager.a on smithi086 2024-04-02T04:49:25.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:49:25 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:49:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:49:26.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:25 smithi086 ceph-mon[24811]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:26.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:25 smithi086 ceph-mon[29505]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:26.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:25 smithi192 ceph-mon[25697]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:26.433 INFO:teuthology.orchestra.run.smithi086.stdout:77309411360 2024-04-02T04:49:26.434 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.2 2024-04-02T04:49:27.993 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:27 smithi086 ceph-mon[29505]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:27.994 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:27 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:27.994 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:27 smithi086 ceph-mon[24811]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:27.994 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:27 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:27 smithi192 ceph-mon[25697]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:27 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:49:29.588 INFO:teuthology.orchestra.run.smithi086.stderr:Inferring config /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T04:49:30.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:29 smithi086 ceph-mon[24811]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:30.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:29 smithi086 ceph-mon[29505]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:30.130 INFO:teuthology.orchestra.run.smithi086.stdout:55834574886 2024-04-02T04:49:30.131 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.1 2024-04-02T04:49:30.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:29 smithi192 ceph-mon[25697]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:30.377 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:49:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:49:30] "GET /metrics HTTP/1.1" 200 191479 "" "Prometheus/2.33.4" 2024-04-02T04:49:30.894 INFO:teuthology.orchestra.run.smithi086.stderr:Inferring config /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T04:49:31.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:31 smithi192 ceph-mon[25697]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:32.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:31 smithi086 ceph-mon[24811]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:32.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:31 smithi086 ceph-mon[29505]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:32.769 INFO:teuthology.orchestra.run.smithi086.stdout:231928233994 2024-04-02T04:49:32.769 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.7 2024-04-02T04:49:34.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:33 smithi086 ceph-mon[24811]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:34.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:33 smithi086 ceph-mon[29505]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:34.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:33 smithi192 ceph-mon[25697]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:34.200 INFO:teuthology.orchestra.run.smithi086.stdout:163208757269 2024-04-02T04:49:34.200 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.5 2024-04-02T04:49:35.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:49:35 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:49:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:49:36.066 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:35 smithi086 ceph-mon[29505]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:36.067 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:35 smithi086 ceph-mon[24811]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:36.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:35 smithi192 ceph-mon[25697]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:36.829 INFO:teuthology.orchestra.run.smithi086.stderr:Inferring config /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T04:49:37.823 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:37 smithi086 ceph-mon[29505]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:37.824 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:37 smithi086 ceph-mon[24811]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:38.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:37 smithi192 ceph-mon[25697]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:38.854 INFO:teuthology.orchestra.run.smithi086.stderr:Inferring config /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T04:49:38.928 INFO:teuthology.orchestra.run.smithi086.stdout:107374182430 2024-04-02T04:49:38.928 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.3 2024-04-02T04:49:39.761 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:39 smithi086 ceph-mon[29505]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:39.762 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:39 smithi086 ceph-mon[24811]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:40.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:39 smithi192 ceph-mon[25697]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:40.334 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:49:40 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:49:40] "GET /metrics HTTP/1.1" 200 191479 "" "Prometheus/2.33.4" 2024-04-02T04:49:40.571 INFO:teuthology.orchestra.run.smithi086.stderr:Inferring config /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T04:49:41.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:41 smithi192 ceph-mon[25697]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:42.019 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:41 smithi086 ceph-mon[29505]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:42.019 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:41 smithi086 ceph-mon[24811]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:42.311 INFO:teuthology.orchestra.run.smithi086.stdout:34359738414 2024-04-02T04:49:42.311 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.0 2024-04-02T04:49:43.781 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:43 smithi086 ceph-mon[29505]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:43.781 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:43 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1185241805' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-02T04:49:43.781 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:43 smithi086 ceph-mon[24811]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:43.781 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:43 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1185241805' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-02T04:49:44.157 INFO:teuthology.orchestra.run.smithi086.stdout:128849018907 2024-04-02T04:49:44.158 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.4 2024-04-02T04:49:44.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:43 smithi192 ceph-mon[25697]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:44.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:43 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1185241805' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-02T04:49:44.260 INFO:teuthology.orchestra.run.smithi086.stdout:163208757270 2024-04-02T04:49:44.826 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757269 got 163208757270 for osd.5 2024-04-02T04:49:44.827 DEBUG:teuthology.parallel:result is None 2024-04-02T04:49:45.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:49:45 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:49:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:49:45.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:45 smithi086 ceph-mon[24811]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:45.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:45 smithi086 ceph-mon[29505]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:46.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:45 smithi192 ceph-mon[25697]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:46.624 INFO:teuthology.orchestra.run.smithi086.stdout:77309411364 2024-04-02T04:49:46.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:46 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/2210980902' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-02T04:49:46.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:46 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/2210980902' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-02T04:49:47.129 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411360 got 77309411364 for osd.2 2024-04-02T04:49:47.130 DEBUG:teuthology.parallel:result is None 2024-04-02T04:49:47.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:46 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/2210980902' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-02T04:49:48.034 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:47 smithi086 ceph-mon[29505]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:48.034 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:47 smithi086 ceph-mon[24811]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:48.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:47 smithi192 ceph-mon[25697]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:48.647 INFO:teuthology.orchestra.run.smithi086.stderr:Inferring config /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T04:49:49.029 INFO:teuthology.orchestra.run.smithi086.stdout:197568495635 2024-04-02T04:49:49.029 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.6 2024-04-02T04:49:49.493 INFO:teuthology.orchestra.run.smithi086.stderr:Inferring config /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T04:49:49.680 INFO:teuthology.orchestra.run.smithi086.stdout:231928233999 2024-04-02T04:49:49.830 INFO:teuthology.orchestra.run.smithi086.stderr:Inferring config /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T04:49:49.954 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:49 smithi086 ceph-mon[29505]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:49.955 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:49 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/731948341' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-02T04:49:49.955 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:49 smithi086 ceph-mon[24811]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:49.955 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:49 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/731948341' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-02T04:49:50.029 INFO:teuthology.orchestra.run.smithi086.stdout:55834574891 2024-04-02T04:49:50.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:49 smithi192 ceph-mon[25697]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:50.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:49 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/731948341' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-02T04:49:50.333 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:49:50 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:49:50] "GET /metrics HTTP/1.1" 200 191465 "" "Prometheus/2.33.4" 2024-04-02T04:49:51.065 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:50 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1884857792' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-02T04:49:51.066 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:50 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1884857792' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-02T04:49:51.090 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928233994 got 231928233999 for osd.7 2024-04-02T04:49:51.090 DEBUG:teuthology.parallel:result is None 2024-04-02T04:49:51.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:50 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1884857792' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-02T04:49:51.912 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:51 smithi086 ceph-mon[29505]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:51.912 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:51 smithi086 ceph-mon[24811]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:52.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:51 smithi192 ceph-mon[25697]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:52.678 INFO:teuthology.orchestra.run.smithi086.stderr:Inferring config /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T04:49:52.705 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574886 got 55834574891 for osd.1 2024-04-02T04:49:52.705 DEBUG:teuthology.parallel:result is None 2024-04-02T04:49:53.999 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:53 smithi192 ceph-mon[25697]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:54.067 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:53 smithi086 ceph-mon[29505]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:54.067 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:53 smithi086 ceph-mon[24811]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:55.427 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:49:55 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:49:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:49:55.995 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:55 smithi086 ceph-mon[24811]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:55.996 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:55 smithi086 ceph-mon[29505]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:56.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:55 smithi192 ceph-mon[25697]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:56.604 INFO:teuthology.orchestra.run.smithi086.stdout:128849018910 2024-04-02T04:49:56.927 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:56 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/3051343939' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-02T04:49:56.928 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:56 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/2312260249' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-02T04:49:56.928 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:56 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:49:56.928 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:56 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:49:56.928 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:56 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/3051343939' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-02T04:49:56.928 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:56 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/2312260249' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-02T04:49:56.928 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:56 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:49:56.928 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:56 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:49:56.950 INFO:teuthology.orchestra.run.smithi086.stdout:107374182434 2024-04-02T04:49:57.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:56 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/3051343939' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-02T04:49:57.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:56 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/2312260249' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-02T04:49:57.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:56 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:49:57.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:56 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:49:57.484 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182430 got 107374182434 for osd.3 2024-04-02T04:49:57.484 DEBUG:teuthology.parallel:result is None 2024-04-02T04:49:57.580 INFO:teuthology.orchestra.run.smithi086.stdout:197568495637 2024-04-02T04:49:58.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:57 smithi086 ceph-mon[24811]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:58.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:57 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/3243758065' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-02T04:49:58.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:57 smithi086 ceph-mon[29505]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:58.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:57 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/3243758065' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-02T04:49:58.160 INFO:teuthology.orchestra.run.smithi086.stdout:34359738417 2024-04-02T04:49:58.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:57 smithi192 ceph-mon[25697]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:58.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:57 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/3243758065' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-02T04:49:58.432 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495635 got 197568495637 for osd.6 2024-04-02T04:49:58.432 DEBUG:teuthology.parallel:result is None 2024-04-02T04:49:58.745 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:49:58 smithi086 podman[47022]: 2024-04-02 04:49:58.616787224 +0000 UTC m=+2.729687261 container create b29bd562b181718d87bc5b34dc152e29ff7569f218c03d6ba99f7da7f3d25518 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T04:49:59.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:58 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1920521532' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-02T04:49:59.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:58 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1920521532' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-02T04:49:59.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:58 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1920521532' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-02T04:49:59.525 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:49:59 smithi086 podman[47022]: 2024-04-02 04:49:59.086975082 +0000 UTC m=+3.199875076 container init b29bd562b181718d87bc5b34dc152e29ff7569f218c03d6ba99f7da7f3d25518 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T04:49:59.526 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:49:59 smithi086 podman[47022]: 2024-04-02 04:49:59.089967734 +0000 UTC m=+3.202867737 container start b29bd562b181718d87bc5b34dc152e29ff7569f218c03d6ba99f7da7f3d25518 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T04:49:59.526 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:49:59 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[47190]: level=info ts=2024-04-02T04:49:59.101Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-04-02T04:49:59.526 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:49:59 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[47190]: level=info ts=2024-04-02T04:49:59.101Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-04-02T04:49:59.526 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:49:59 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[47190]: level=info ts=2024-04-02T04:49:59.102Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.86 port=9094 2024-04-02T04:49:59.526 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:49:59 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[47190]: level=info ts=2024-04-02T04:49:59.104Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-02T04:49:59.526 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:49:59 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[47190]: level=info ts=2024-04-02T04:49:59.129Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-02T04:49:59.526 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:49:59 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[47190]: level=info ts=2024-04-02T04:49:59.129Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-02T04:49:59.526 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:49:59 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[47190]: level=info ts=2024-04-02T04:49:59.131Z caller=main.go:518 msg=Listening address=:9093 2024-04-02T04:49:59.526 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:49:59 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[47190]: level=info ts=2024-04-02T04:49:59.131Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-04-02T04:49:59.552 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738414 got 34359738417 for osd.0 2024-04-02T04:49:59.552 DEBUG:teuthology.parallel:result is None 2024-04-02T04:49:59.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:49:59 smithi086 ceph-mon[24811]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:59.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:49:59 smithi086 ceph-mon[29505]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:49:59.905 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018907 got 128849018910 for osd.4 2024-04-02T04:49:59.905 DEBUG:teuthology.parallel:result is None 2024-04-02T04:49:59.906 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-02T04:49:59.906 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph pg dump --format=json 2024-04-02T04:50:00.122 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:49:59 smithi192 ceph-mon[25697]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:50:00.131 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:50:00 smithi086 bash[47022]: b29bd562b181718d87bc5b34dc152e29ff7569f218c03d6ba99f7da7f3d25518 2024-04-02T04:50:00.131 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:50:00 smithi086 systemd[1]: Started Ceph alertmanager.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T04:50:00.138 INFO:teuthology.orchestra.run.smithi086.stderr:Inferring config /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T04:50:00.411 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:50:00 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:50:00] "GET /metrics HTTP/1.1" 200 191470 "" "Prometheus/2.33.4" 2024-04-02T04:50:00.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:00 smithi192 ceph-mon[25697]: overall HEALTH_OK 2024-04-02T04:50:00.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:00 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:00.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:00 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:50:00.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:00 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:50:00.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:00 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:50:01.010 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:00 smithi086 ceph-mon[29505]: overall HEALTH_OK 2024-04-02T04:50:01.010 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:00 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:01.010 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:00 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:50:01.010 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:00 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:50:01.010 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:00 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:50:01.010 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:00 smithi086 ceph-mon[24811]: overall HEALTH_OK 2024-04-02T04:50:01.011 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:00 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:01.011 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:00 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:50:01.011 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:00 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:50:01.011 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:00 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:50:01.333 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:50:01 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[47190]: level=info ts=2024-04-02T04:50:01.104Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000460525s 2024-04-02T04:50:01.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:01 smithi192 ceph-mon[25697]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:50:01.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:01 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:02.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:01 smithi086 ceph-mon[29505]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:50:02.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:01 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:02.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:01 smithi086 ceph-mon[24811]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:50:02.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:01 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:02.667 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-02T04:50:02.668 INFO:teuthology.orchestra.run.smithi086.stderr:dumped all 2024-04-02T04:50:03.982 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:03 smithi086 ceph-mon[29505]: from='client.14592 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T04:50:03.982 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:03 smithi086 ceph-mon[29505]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:50:03.982 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:03 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:03.982 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:03 smithi086 ceph-mon[24811]: from='client.14592 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T04:50:03.982 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:03 smithi086 ceph-mon[24811]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:50:03.982 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:03 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:04.011 INFO:teuthology.orchestra.run.smithi086.stdout:{"pg_ready":true,"pg_map":{"version":35,"stamp":"2024-04-02T04:50:00.661381+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48888,"kb_used_data":6072,"kb_used_omap":0,"kb_used_meta":42752,"kb_avail":749912328,"statfs":{"total":767960285184,"available":767910223872,"internally_reserved":0,"allocated":6217728,"data_stored":3487276,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43778048},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002499"},"pg_stats":[{"pgid":"1.0","version":"59'128","reported_seq":266,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-04-02T04:48:56.728448+0000","last_change":"2024-04-02T04:48:44.642498+0000","last_active":"2024-04-02T04:48:56.728448+0000","last_peered":"2024-04-02T04:48:56.728448+0000","last_clean":"2024-04-02T04:48:56.728448+0000","last_became_active":"2024-04-02T04:48:44.642096+0000","last_became_peered":"2024-04-02T04:48:44.642096+0000","last_unstale":"2024-04-02T04:48:56.728448+0000","last_undegraded":"2024-04-02T04:48:56.728448+0000","last_fullsized":"2024-04-02T04:48:56.728448+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-02T04:46:48.377389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-02T04:46:48.377389+0000","last_clean_scrub_stamp":"2024-04-02T04:46:48.377389+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-03T08:25:41.250278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234001,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6088,"kb_used_data":1088,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739064,"statfs":{"total":95995035648,"available":95988801536,"internally_reserved":0,"allocated":1114112,"data_stored":766769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 04:49:42 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.51200000000000001}]},{"osd":1,"last update":"Tue Apr 2 04:49:42 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.40000000000000002}]},{"osd":2,"last update":"Tue Apr 2 04:49:42 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.48199999999999998}]},{"osd":3,"last update":"Tue Apr 2 04:49:42 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.127,"5min":1.127,"15min":1.127},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":10.722,"5min":10.722,"15min":10.722},"last":0.54500000000000004}]},{"osd":4,"last update":"Tue Apr 2 04:49:42 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.41399999999999998}]},{"osd":5,"last update":"Tue Apr 2 04:49:42 2024","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.46500000000000002}]},{"osd":6,"last update":"Tue Apr 2 04:49:42 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.499}]}]},{"osd":6,"up_from":46,"seq":197568495638,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6148,"kb_used_data":1084,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739004,"statfs":{"total":95995035648,"available":95988740096,"internally_reserved":0,"allocated":1110016,"data_stored":766446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 04:49:19 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.442},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.47299999999999998}]},{"osd":1,"last update":"Tue Apr 2 04:49:19 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":1.2889999999999999},{"interface":"front","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.34599999999999997}]},{"osd":2,"last update":"Tue Apr 2 04:49:19 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.38700000000000001}]},{"osd":3,"last update":"Tue Apr 2 04:49:19 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.495},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.505}]},{"osd":4,"last update":"Tue Apr 2 04:49:19 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.55800000000000005}]},{"osd":5,"last update":"Tue Apr 2 04:49:19 2024","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.52800000000000002}]},{"osd":7,"last update":"Tue Apr 2 04:49:41 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.58499999999999996}]}]},{"osd":1,"up_from":13,"seq":55834574893,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6328,"kb_used_data":560,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738824,"statfs":{"total":95995035648,"available":95988555776,"internally_reserved":0,"allocated":573440,"data_stored":237311,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 04:49:31 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.35399999999999998,"15min":0.32600000000000001},"min":{"1min":0.23999999999999999,"5min":0.223,"15min":0.223},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.34999999999999998,"15min":0.32000000000000001},"min":{"1min":0.25600000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.34999999999999998}]},{"osd":2,"last update":"Tue Apr 2 04:49:56 2024","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.41199999999999998,"15min":0.40999999999999998},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.57499999999999996,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.39600000000000002,"15min":0.39200000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"last":0.31900000000000001}]},{"osd":3,"last update":"Tue Apr 2 04:49:13 2024","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.442,"15min":0.442},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.63300000000000001,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.33200000000000002}]},{"osd":4,"last update":"Tue Apr 2 04:49:39 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.60699999999999998,"15min":0.60799999999999998},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.75900000000000001,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.624,"5min":0.59499999999999997,"15min":0.58999999999999997},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.93500000000000005,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.56599999999999995}]},{"osd":5,"last update":"Tue Apr 2 04:49:56 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.63700000000000001,"15min":0.63900000000000001},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.88500000000000001,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.65900000000000003,"15min":0.66300000000000003},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.85899999999999999,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.46300000000000002}]},{"osd":6,"last update":"Tue Apr 2 04:49:18 2024","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.99299999999999999,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.996,"5min":0.996,"15min":0.996},"last":0.51600000000000001}]},{"osd":7,"last update":"Tue Apr 2 04:49:45 2024","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.496},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.3,"5min":1.3,"15min":1.3},"last":0.57299999999999995}]}]},{"osd":0,"up_from":8,"seq":34359738418,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6920,"kb_used_data":1088,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738232,"statfs":{"total":95995035648,"available":95987949568,"internally_reserved":0,"allocated":1114112,"data_stored":766769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Tue Apr 2 04:49:31 2024","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.34799999999999998,"15min":0.32900000000000001},"min":{"1min":0.24299999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.497,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.36899999999999999,"15min":0.35399999999999998},"min":{"1min":0.249,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.45000000000000001,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.30099999999999999}]},{"osd":2,"last update":"Tue Apr 2 04:49:52 2024","interfaces":[{"interface":"back","average":{"1min":0.34399999999999997,"5min":0.38400000000000001,"15min":0.39200000000000002},"min":{"1min":0.255,"5min":0.217,"15min":0.217},"max":{"1min":0.503,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.41699999999999998,"15min":0.41699999999999998},"min":{"1min":0.315,"5min":0.253,"15min":0.253},"max":{"1min":0.626,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.36099999999999999}]},{"osd":3,"last update":"Tue Apr 2 04:49:18 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.42699999999999999,"15min":0.42899999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.48099999999999998,"15min":0.48499999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.62,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.32200000000000001}]},{"osd":4,"last update":"Tue Apr 2 04:49:34 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.61099999999999999,"15min":0.61899999999999999},"min":{"1min":0.46999999999999997,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.68899999999999995,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.56299999999999994,"5min":1.099,"15min":1.1879999999999999},"min":{"1min":0.44,"5min":0.433,"15min":0.433},"max":{"1min":0.92800000000000005,"5min":13.076000000000001,"15min":13.076000000000001},"last":0.54200000000000004}]},{"osd":5,"last update":"Tue Apr 2 04:48:58 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.5}]},{"osd":6,"last update":"Tue Apr 2 04:49:18 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.53300000000000003}]},{"osd":7,"last update":"Tue Apr 2 04:49:42 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.46800000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411367,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6272,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738880,"statfs":{"total":95995035648,"available":95988613120,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 04:49:54 2024","interfaces":[{"interface":"back","average":{"1min":0.35099999999999998,"5min":0.38300000000000001,"15min":0.39100000000000001},"min":{"1min":0.27800000000000002,"5min":0.22,"15min":0.22},"max":{"1min":0.55200000000000005,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.34499999999999997,"5min":0.373,"15min":0.378},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.53200000000000003,"5min":0.55100000000000005,"15min":0.55100000000000005},"last":0.36199999999999999}]},{"osd":1,"last update":"Tue Apr 2 04:49:54 2024","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.38200000000000001,"15min":0.38},"min":{"1min":0.27300000000000002,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.51500000000000001,"5min":0.56699999999999995,"15min":0.56699999999999995},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.40999999999999998,"15min":0.40799999999999997},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.56799999999999995,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":0.34399999999999997}]},{"osd":3,"last update":"Tue Apr 2 04:49:12 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.46800000000000003,"15min":0.47299999999999998},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.44900000000000001,"15min":0.45600000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"last":0.46400000000000002}]},{"osd":4,"last update":"Tue Apr 2 04:49:39 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60799999999999998,"15min":0.60899999999999999},"min":{"1min":0.36499999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.79100000000000004,"5min":1.0389999999999999,"15min":1.0389999999999999},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.58299999999999996,"15min":0.58399999999999996},"min":{"1min":0.42099999999999999,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.73899999999999999,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.59899999999999998}]},{"osd":5,"last update":"Tue Apr 2 04:48:58 2024","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":0.95099999999999996,"15min":0.95099999999999996},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":7.702,"5min":7.702,"15min":7.702},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.52200000000000002}]},{"osd":6,"last update":"Tue Apr 2 04:49:18 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.49199999999999999}]},{"osd":7,"last update":"Tue Apr 2 04:49:42 2024","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":0.97599999999999998,"15min":0.97599999999999998},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":7.6840000000000002,"5min":7.6840000000000002,"15min":7.6840000000000002},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.623}]}]},{"osd":3,"up_from":25,"seq":107374182435,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5688,"kb_used_data":560,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739464,"statfs":{"total":95995035648,"available":95989211136,"internally_reserved":0,"allocated":573440,"data_stored":237311,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 04:49:16 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.42199999999999999,"15min":0.42699999999999999},"min":{"1min":0.29799999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.57599999999999996,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.42799999999999999,"15min":0.432},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.63200000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.34100000000000003}]},{"osd":1,"last update":"Tue Apr 2 04:49:16 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.45700000000000002,"15min":0.46000000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.57899999999999996,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.45300000000000001,"15min":0.45800000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.59399999999999997,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.32700000000000001}]},{"osd":2,"last update":"Tue Apr 2 04:49:16 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.47599999999999998,"15min":0.48199999999999998},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.65000000000000002,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.46899999999999997,"15min":0.47699999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.56000000000000005,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.317}]},{"osd":4,"last update":"Tue Apr 2 04:49:36 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.58899999999999997,"15min":0.59299999999999997},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.60099999999999998,"15min":0.60399999999999998},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.52300000000000002}]},{"osd":5,"last update":"Tue Apr 2 04:48:58 2024","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.51500000000000001}]},{"osd":6,"last update":"Tue Apr 2 04:49:17 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.442}]},{"osd":7,"last update":"Tue Apr 2 04:49:41 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.45100000000000001}]}]},{"osd":4,"up_from":30,"seq":128849018911,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5756,"kb_used_data":564,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739396,"statfs":{"total":95995035648,"available":95989141504,"internally_reserved":0,"allocated":577536,"data_stored":237576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 04:49:36 2024","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.51900000000000002,"15min":0.52500000000000002},"min":{"1min":0.32500000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.68899999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.499,"5min":0.51800000000000002,"15min":0.52100000000000002},"min":{"1min":0.33200000000000002,"5min":0.254,"15min":0.254},"max":{"1min":0.70299999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.48099999999999998}]},{"osd":1,"last update":"Tue Apr 2 04:49:36 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.52200000000000002,"15min":0.52300000000000002},"min":{"1min":0.35699999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.65800000000000003,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.54200000000000004,"15min":0.55400000000000005},"min":{"1min":0.38100000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.59699999999999998,"5min":1.3029999999999999,"15min":1.3029999999999999},"last":0.504}]},{"osd":2,"last update":"Tue Apr 2 04:49:36 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.57299999999999995,"15min":0.58399999999999996},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.68899999999999995,"5min":1.3759999999999999,"15min":1.3759999999999999},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.56699999999999995,"15min":0.57299999999999995},"min":{"1min":0.36299999999999999,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.78100000000000003,"5min":1.3839999999999999,"15min":1.3839999999999999},"last":0.51800000000000002}]},{"osd":3,"last update":"Tue Apr 2 04:49:36 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.57599999999999996,"15min":0.58099999999999996},"min":{"1min":0.42399999999999999,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.72399999999999998,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.57999999999999996,"15min":0.58699999999999997},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.71899999999999997,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":0.56599999999999995}]},{"osd":5,"last update":"Tue Apr 2 04:48:54 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.55200000000000005}]},{"osd":6,"last update":"Tue Apr 2 04:49:18 2024","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.373},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.54000000000000004}]},{"osd":7,"last update":"Tue Apr 2 04:49:43 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.443}]}]},{"osd":5,"up_from":38,"seq":163208757274,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5688,"kb_used_data":560,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739464,"statfs":{"total":95995035648,"available":95989211136,"internally_reserved":0,"allocated":573440,"data_stored":237253,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 04:49:00 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.46800000000000003},{"interface":"front","average":{"1min":1.224,"5min":1.224,"15min":1.224},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":14.91,"5min":14.91,"15min":14.91},"last":0.629}]},{"osd":1,"last update":"Tue Apr 2 04:49:00 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.58499999999999996}]},{"osd":2,"last update":"Tue Apr 2 04:49:00 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.62},{"interface":"front","average":{"1min":1.2410000000000001,"5min":1.2410000000000001,"15min":1.2410000000000001},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":14.961,"5min":14.961,"15min":14.961},"last":0.55500000000000005}]},{"osd":3,"last update":"Tue Apr 2 04:49:00 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.93400000000000005,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.234,"5min":1.234,"15min":1.234},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":14.933,"5min":14.933,"15min":14.933},"last":0.52100000000000002}]},{"osd":4,"last update":"Tue Apr 2 04:49:00 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"last":0.59699999999999998}]},{"osd":6,"last update":"Tue Apr 2 04:49:17 2024","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.57099999999999995}]},{"osd":7,"last update":"Tue Apr 2 04:49:40 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.60699999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-02T04:50:04.012 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph pg dump --format=json 2024-04-02T04:50:04.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:03 smithi192 ceph-mon[25697]: from='client.14592 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T04:50:04.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:03 smithi192 ceph-mon[25697]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:50:04.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:03 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:04.663 INFO:teuthology.orchestra.run.smithi086.stderr:Inferring config /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T04:50:05.427 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:50:05 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:50:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:50:05.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:05 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:05.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:05 smithi192 ceph-mon[25697]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:50:05.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:05 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:05.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:05 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:05.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:05 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:05.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:05 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-02T04:50:05.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:05 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:05.981 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:05 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:05.981 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:05 smithi086 ceph-mon[24811]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:50:05.981 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:05 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:05.981 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:05 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:05.982 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:05 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:05.982 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:05 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-02T04:50:05.982 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:05 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:05.982 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:05 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:05.982 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:05 smithi086 ceph-mon[29505]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:50:05.982 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:05 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:05.982 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:05 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:05.982 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:05 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:05.982 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:05 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-02T04:50:05.982 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:05 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:06.628 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-02T04:50:06.630 INFO:teuthology.orchestra.run.smithi086.stderr:dumped all 2024-04-02T04:50:06.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:06 smithi192 ceph-mon[25697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-02T04:50:06.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:06 smithi192 ceph-mon[25697]: Deploying daemon grafana.a on smithi192 2024-04-02T04:50:06.952 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:06 smithi086 ceph-mon[29505]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-02T04:50:06.952 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:06 smithi086 ceph-mon[29505]: Deploying daemon grafana.a on smithi192 2024-04-02T04:50:06.952 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:06 smithi086 ceph-mon[24811]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-02T04:50:06.953 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:06 smithi086 ceph-mon[24811]: Deploying daemon grafana.a on smithi192 2024-04-02T04:50:07.302 INFO:teuthology.orchestra.run.smithi086.stdout:{"pg_ready":true,"pg_map":{"version":37,"stamp":"2024-04-02T04:50:04.663219+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48888,"kb_used_data":6072,"kb_used_omap":0,"kb_used_meta":42752,"kb_avail":749912328,"statfs":{"total":767960285184,"available":767910223872,"internally_reserved":0,"allocated":6217728,"data_stored":3487276,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43778048},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003609"},"pg_stats":[{"pgid":"1.0","version":"59'128","reported_seq":266,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-04-02T04:48:56.728448+0000","last_change":"2024-04-02T04:48:44.642498+0000","last_active":"2024-04-02T04:48:56.728448+0000","last_peered":"2024-04-02T04:48:56.728448+0000","last_clean":"2024-04-02T04:48:56.728448+0000","last_became_active":"2024-04-02T04:48:44.642096+0000","last_became_peered":"2024-04-02T04:48:44.642096+0000","last_unstale":"2024-04-02T04:48:56.728448+0000","last_undegraded":"2024-04-02T04:48:56.728448+0000","last_fullsized":"2024-04-02T04:48:56.728448+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-02T04:46:48.377389+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-02T04:46:48.377389+0000","last_clean_scrub_stamp":"2024-04-02T04:46:48.377389+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-03T08:25:41.250278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234002,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6088,"kb_used_data":1088,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739064,"statfs":{"total":95995035648,"available":95988801536,"internally_reserved":0,"allocated":1114112,"data_stored":766769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 04:49:42 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.46300000000000002}]},{"osd":1,"last update":"Tue Apr 2 04:49:42 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.40100000000000002}]},{"osd":2,"last update":"Tue Apr 2 04:49:42 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.434},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.41799999999999998}]},{"osd":3,"last update":"Tue Apr 2 04:49:42 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.41099999999999998},{"interface":"front","average":{"1min":1.127,"5min":1.127,"15min":1.127},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":10.722,"5min":10.722,"15min":10.722},"last":0.502}]},{"osd":4,"last update":"Tue Apr 2 04:49:42 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.37},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.45700000000000002}]},{"osd":5,"last update":"Tue Apr 2 04:49:42 2024","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.42599999999999999}]},{"osd":6,"last update":"Tue Apr 2 04:49:42 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.38200000000000001}]}]},{"osd":6,"up_from":46,"seq":197568495639,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6148,"kb_used_data":1084,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739004,"statfs":{"total":95995035648,"available":95988740096,"internally_reserved":0,"allocated":1110016,"data_stored":766446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 04:49:19 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.46700000000000003}]},{"osd":1,"last update":"Tue Apr 2 04:49:19 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.45100000000000001}]},{"osd":2,"last update":"Tue Apr 2 04:49:19 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.59999999999999998}]},{"osd":3,"last update":"Tue Apr 2 04:49:19 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.57799999999999996}]},{"osd":4,"last update":"Tue Apr 2 04:49:19 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.61199999999999999}]},{"osd":5,"last update":"Tue Apr 2 04:49:19 2024","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.63200000000000001}]},{"osd":7,"last update":"Tue Apr 2 04:49:41 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.628},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.64700000000000002}]}]},{"osd":1,"up_from":13,"seq":55834574894,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6328,"kb_used_data":560,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738824,"statfs":{"total":95995035648,"available":95988555776,"internally_reserved":0,"allocated":573440,"data_stored":237311,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 04:49:31 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.35399999999999998,"15min":0.32600000000000001},"min":{"1min":0.23999999999999999,"5min":0.223,"15min":0.223},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.34999999999999998,"15min":0.32000000000000001},"min":{"1min":0.25600000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.28699999999999998}]},{"osd":2,"last update":"Tue Apr 2 04:49:56 2024","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.41199999999999998,"15min":0.40999999999999998},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.57499999999999996,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.39600000000000002,"15min":0.39200000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"last":0.59899999999999998}]},{"osd":3,"last update":"Tue Apr 2 04:49:13 2024","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.442,"15min":0.442},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.63300000000000001,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.61399999999999999}]},{"osd":4,"last update":"Tue Apr 2 04:49:39 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.60699999999999998,"15min":0.60799999999999998},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.75900000000000001,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.624,"5min":0.59499999999999997,"15min":0.58999999999999997},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.93500000000000005,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.70299999999999996}]},{"osd":5,"last update":"Tue Apr 2 04:49:56 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.63700000000000001,"15min":0.63900000000000001},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.88500000000000001,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.65900000000000003,"15min":0.66300000000000003},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.85899999999999999,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.63100000000000001}]},{"osd":6,"last update":"Tue Apr 2 04:49:18 2024","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.99299999999999999,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.996,"5min":0.996,"15min":0.996},"last":0.74399999999999999}]},{"osd":7,"last update":"Tue Apr 2 04:49:45 2024","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.3,"5min":1.3,"15min":1.3},"last":0.72899999999999998}]}]},{"osd":0,"up_from":8,"seq":34359738419,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6920,"kb_used_data":1088,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738232,"statfs":{"total":95995035648,"available":95987949568,"internally_reserved":0,"allocated":1114112,"data_stored":766769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Tue Apr 2 04:49:31 2024","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.34799999999999998,"15min":0.32900000000000001},"min":{"1min":0.24299999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.497,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.36899999999999999,"15min":0.35399999999999998},"min":{"1min":0.249,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.45000000000000001,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.42199999999999999}]},{"osd":2,"last update":"Tue Apr 2 04:49:52 2024","interfaces":[{"interface":"back","average":{"1min":0.34399999999999997,"5min":0.38400000000000001,"15min":0.39200000000000002},"min":{"1min":0.255,"5min":0.217,"15min":0.217},"max":{"1min":0.503,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.316},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.41699999999999998,"15min":0.41699999999999998},"min":{"1min":0.315,"5min":0.253,"15min":0.253},"max":{"1min":0.626,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.437}]},{"osd":3,"last update":"Tue Apr 2 04:49:18 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.42699999999999999,"15min":0.42899999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.48099999999999998,"15min":0.48499999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.62,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.39300000000000002}]},{"osd":4,"last update":"Tue Apr 2 04:49:34 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.61099999999999999,"15min":0.61899999999999999},"min":{"1min":0.46999999999999997,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.68899999999999995,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.56299999999999994,"5min":1.099,"15min":1.1879999999999999},"min":{"1min":0.44,"5min":0.433,"15min":0.433},"max":{"1min":0.92800000000000005,"5min":13.076000000000001,"15min":13.076000000000001},"last":0.59499999999999997}]},{"osd":5,"last update":"Tue Apr 2 04:50:00 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.42799999999999999,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.79100000000000004,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.56899999999999995,"15min":0.56599999999999995},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.627}]},{"osd":6,"last update":"Tue Apr 2 04:49:18 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.61699999999999999}]},{"osd":7,"last update":"Tue Apr 2 04:49:42 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.60499999999999998}]}]},{"osd":2,"up_from":18,"seq":77309411368,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6272,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738880,"statfs":{"total":95995035648,"available":95988613120,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 04:49:54 2024","interfaces":[{"interface":"back","average":{"1min":0.35099999999999998,"5min":0.38300000000000001,"15min":0.39100000000000001},"min":{"1min":0.27800000000000002,"5min":0.22,"15min":0.22},"max":{"1min":0.55200000000000005,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.19900000000000001},{"interface":"front","average":{"1min":0.34499999999999997,"5min":0.373,"15min":0.378},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.53200000000000003,"5min":0.55100000000000005,"15min":0.55100000000000005},"last":0.377}]},{"osd":1,"last update":"Tue Apr 2 04:49:54 2024","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.38200000000000001,"15min":0.38},"min":{"1min":0.27300000000000002,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.51500000000000001,"5min":0.56699999999999995,"15min":0.56699999999999995},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.40999999999999998,"15min":0.40799999999999997},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.56799999999999995,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":0.28299999999999997}]},{"osd":3,"last update":"Tue Apr 2 04:49:12 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.46800000000000003,"15min":0.47299999999999998},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.317},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.44900000000000001,"15min":0.45600000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"last":0.247}]},{"osd":4,"last update":"Tue Apr 2 04:49:39 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60799999999999998,"15min":0.60899999999999999},"min":{"1min":0.36499999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.79100000000000004,"5min":1.0389999999999999,"15min":1.0389999999999999},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.58299999999999996,"15min":0.58399999999999996},"min":{"1min":0.42099999999999999,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.73899999999999999,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.35199999999999998}]},{"osd":5,"last update":"Tue Apr 2 04:50:01 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.878,"15min":0.92600000000000005},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.83899999999999997,"5min":7.702,"15min":7.702},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.628,"5min":0.59799999999999998,"15min":0.59299999999999997},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.83499999999999996,"5min":0.878,"15min":0.878},"last":0.41699999999999998}]},{"osd":6,"last update":"Tue Apr 2 04:49:18 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.42999999999999999}]},{"osd":7,"last update":"Tue Apr 2 04:49:42 2024","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":0.97599999999999998,"15min":0.97599999999999998},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":7.6840000000000002,"5min":7.6840000000000002,"15min":7.6840000000000002},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.41199999999999998}]}]},{"osd":3,"up_from":25,"seq":107374182436,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5688,"kb_used_data":560,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739464,"statfs":{"total":95995035648,"available":95989211136,"internally_reserved":0,"allocated":573440,"data_stored":237311,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 04:49:16 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.42199999999999999,"15min":0.42699999999999999},"min":{"1min":0.29799999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.57599999999999996,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.42799999999999999,"15min":0.432},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.63200000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.28199999999999997}]},{"osd":1,"last update":"Tue Apr 2 04:49:16 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.45700000000000002,"15min":0.46000000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.57899999999999996,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.315},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.45300000000000001,"15min":0.45800000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.59399999999999997,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.32100000000000001}]},{"osd":2,"last update":"Tue Apr 2 04:49:16 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.47599999999999998,"15min":0.48199999999999998},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.65000000000000002,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.46899999999999997,"15min":0.47699999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.56000000000000005,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.33200000000000002}]},{"osd":4,"last update":"Tue Apr 2 04:49:36 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.58899999999999997,"15min":0.59299999999999997},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.437},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.60099999999999998,"15min":0.60399999999999998},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.48499999999999999}]},{"osd":5,"last update":"Tue Apr 2 04:50:01 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.61199999999999999,"15min":0.62},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.82299999999999995,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.61799999999999999,"15min":0.621},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.51100000000000001}]},{"osd":6,"last update":"Tue Apr 2 04:49:17 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.56599999999999995}]},{"osd":7,"last update":"Tue Apr 2 04:49:41 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.55300000000000005}]}]},{"osd":4,"up_from":30,"seq":128849018912,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5756,"kb_used_data":564,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739396,"statfs":{"total":95995035648,"available":95989141504,"internally_reserved":0,"allocated":577536,"data_stored":237576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 04:49:36 2024","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.51900000000000002,"15min":0.52500000000000002},"min":{"1min":0.32500000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.68899999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.499,"5min":0.51800000000000002,"15min":0.52100000000000002},"min":{"1min":0.33200000000000002,"5min":0.254,"15min":0.254},"max":{"1min":0.70299999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.69399999999999995}]},{"osd":1,"last update":"Tue Apr 2 04:49:36 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.52200000000000002,"15min":0.52300000000000002},"min":{"1min":0.35699999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.65800000000000003,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.70899999999999996},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.54200000000000004,"15min":0.55400000000000005},"min":{"1min":0.38100000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.59699999999999998,"5min":1.3029999999999999,"15min":1.3029999999999999},"last":0.57899999999999996}]},{"osd":2,"last update":"Tue Apr 2 04:49:36 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.57299999999999995,"15min":0.58399999999999996},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.68899999999999995,"5min":1.3759999999999999,"15min":1.3759999999999999},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.56699999999999995,"15min":0.57299999999999995},"min":{"1min":0.36299999999999999,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.78100000000000003,"5min":1.3839999999999999,"15min":1.3839999999999999},"last":0.749}]},{"osd":3,"last update":"Tue Apr 2 04:49:36 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.57599999999999996,"15min":0.58099999999999996},"min":{"1min":0.42399999999999999,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.72399999999999998,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.57999999999999996,"15min":0.58699999999999997},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.71899999999999997,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":0.68300000000000005}]},{"osd":5,"last update":"Tue Apr 2 04:49:59 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.41899999999999998,"15min":0.40500000000000003},"min":{"1min":0.29999999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.434},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.42799999999999999,"15min":0.40799999999999997},"min":{"1min":0.38800000000000001,"5min":0.31,"15min":0.31},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.46899999999999997}]},{"osd":6,"last update":"Tue Apr 2 04:49:18 2024","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.503},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.53000000000000003}]},{"osd":7,"last update":"Tue Apr 2 04:49:43 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.54700000000000004}]}]},{"osd":5,"up_from":38,"seq":163208757275,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5688,"kb_used_data":560,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739464,"statfs":{"total":95995035648,"available":95989211136,"internally_reserved":0,"allocated":573440,"data_stored":237253,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 04:50:00 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.55600000000000005,"15min":0.56000000000000005},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.66300000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.437},{"interface":"front","average":{"1min":0.53000000000000003,"5min":1.085,"15min":1.177},"min":{"1min":0.35699999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.63,"5min":14.91,"15min":14.91},"last":0.45400000000000001}]},{"osd":1,"last update":"Tue Apr 2 04:50:00 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.54700000000000004,"15min":0.55300000000000005},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.68100000000000005,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.71499999999999997,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.36799999999999999}]},{"osd":2,"last update":"Tue Apr 2 04:50:00 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.55000000000000004,"15min":0.55200000000000005},"min":{"1min":0.40999999999999998,"5min":0.374,"15min":0.374},"max":{"1min":0.64200000000000002,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.56799999999999995,"5min":1.1060000000000001,"15min":1.196},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.68400000000000005,"5min":14.961,"15min":14.961},"last":0.34699999999999998}]},{"osd":3,"last update":"Tue Apr 2 04:50:00 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.56499999999999995,"15min":0.56699999999999995},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.68999999999999995,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.55700000000000005,"5min":1.0980000000000001,"15min":1.1879999999999999},"min":{"1min":0.42599999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.71099999999999997,"5min":14.933,"15min":14.933},"last":0.373}]},{"osd":4,"last update":"Tue Apr 2 04:50:00 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.441,"15min":0.434},"min":{"1min":0.316,"5min":0.31,"15min":0.31},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.433,"15min":0.41999999999999998},"min":{"1min":0.35499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.35399999999999998}]},{"osd":6,"last update":"Tue Apr 2 04:49:17 2024","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.39900000000000002}]},{"osd":7,"last update":"Tue Apr 2 04:49:40 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.39300000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-02T04:50:07.304 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-02T04:50:07.304 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-02T04:50:07.304 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-02T04:50:07.304 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- ceph health --format=json 2024-04-02T04:50:07.517 INFO:teuthology.orchestra.run.smithi086.stderr:Inferring config /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T04:50:07.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:07 smithi192 ceph-mon[25697]: from='client.14598 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T04:50:07.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:07 smithi192 ceph-mon[25697]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:50:07.947 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:07 smithi086 ceph-mon[29505]: from='client.14598 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T04:50:07.947 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:07 smithi086 ceph-mon[29505]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:50:07.948 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:07 smithi086 ceph-mon[24811]: from='client.14598 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T04:50:07.948 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:07 smithi086 ceph-mon[24811]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:50:09.333 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:50:09 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[47190]: level=info ts=2024-04-02T04:50:09.106Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002771615s 2024-04-02T04:50:09.424 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-02T04:50:09.424 INFO:teuthology.orchestra.run.smithi086.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-02T04:50:09.984 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:09 smithi086 ceph-mon[24811]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:50:09.985 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:09 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/217656161' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-02T04:50:09.985 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:09 smithi086 ceph-mon[29505]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:50:09.985 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:09 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/217656161' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-02T04:50:10.008 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-02T04:50:10.008 INFO:tasks.cephadm:Setup complete, yielding 2024-04-02T04:50:10.008 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-02T04:50:10.018 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi086.front.sepia.ceph.com 2024-04-02T04:50:10.018 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- bash -c 'ceph config set mgr mgr/cephadm/use_repo_digest false --force' 2024-04-02T04:50:10.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:09 smithi192 ceph-mon[25697]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:50:10.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:09 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/217656161' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-02T04:50:10.323 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:50:10 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:50:10] "GET /metrics HTTP/1.1" 200 191470 "" "Prometheus/2.33.4" 2024-04-02T04:50:11.887 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:11 smithi086 ceph-mon[29505]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:50:11.888 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:11 smithi086 ceph-mon[24811]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:50:11.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:11 smithi192 ceph-mon[25697]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:50:12.571 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-02T04:50:12.579 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi086.front.sepia.ceph.com 2024-04-02T04:50:12.580 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'radosgw-admin realm create --rgw-realm=r --default' 2024-04-02T04:50:13.127 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:12 smithi086 ceph-mon[29505]: from='client.? ' entity='client.admin' 2024-04-02T04:50:13.127 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:12 smithi086 ceph-mon[24811]: from='client.? ' entity='client.admin' 2024-04-02T04:50:13.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:12 smithi192 ceph-mon[25697]: from='client.? ' entity='client.admin' 2024-04-02T04:50:14.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:13 smithi192 ceph-mon[25697]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:50:14.200 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:13 smithi086 ceph-mon[29505]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:50:14.200 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:13 smithi086 ceph-mon[24811]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:50:15.421 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:50:15 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:50:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:50:15.922 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T04:50:15.922 INFO:teuthology.orchestra.run.smithi086.stdout: "id": "9953c8db-4704-4a82-b62f-70d6707827e1", 2024-04-02T04:50:15.922 INFO:teuthology.orchestra.run.smithi086.stdout: "name": "r", 2024-04-02T04:50:15.922 INFO:teuthology.orchestra.run.smithi086.stdout: "current_period": "ef79ad70-97de-4f81-8759-520eb84d60dd", 2024-04-02T04:50:15.922 INFO:teuthology.orchestra.run.smithi086.stdout: "epoch": 1 2024-04-02T04:50:15.922 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T04:50:16.048 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:15 smithi192 ceph-mon[25697]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:50:16.048 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:15 smithi192 ceph-mon[25697]: osdmap e60: 8 total, 8 up, 8 in 2024-04-02T04:50:16.049 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:15 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/847611354' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-02T04:50:16.193 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:15 smithi086 ceph-mon[24811]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:50:16.194 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:15 smithi086 ceph-mon[24811]: osdmap e60: 8 total, 8 up, 8 in 2024-04-02T04:50:16.194 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:15 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/847611354' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-02T04:50:16.194 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:15 smithi086 ceph-mon[29505]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:50:16.194 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:15 smithi086 ceph-mon[29505]: osdmap e60: 8 total, 8 up, 8 in 2024-04-02T04:50:16.194 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:15 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/847611354' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-02T04:50:16.489 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'radosgw-admin zonegroup create --rgw-zonegroup=default --master --default' 2024-04-02T04:50:17.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:16 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/847611354' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-04-02T04:50:17.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:16 smithi086 ceph-mon[24811]: osdmap e61: 8 total, 8 up, 8 in 2024-04-02T04:50:17.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:16 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/847611354' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-04-02T04:50:17.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:16 smithi086 ceph-mon[29505]: osdmap e61: 8 total, 8 up, 8 in 2024-04-02T04:50:17.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:16 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/847611354' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-04-02T04:50:17.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:16 smithi192 ceph-mon[25697]: osdmap e61: 8 total, 8 up, 8 in 2024-04-02T04:50:17.858 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2024-04-02T04:50:17.858 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2024-04-02T04:50:18.018 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:17 smithi086 ceph-mon[29505]: pgmap v45: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:50:18.018 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:17 smithi086 ceph-mon[29505]: osdmap e62: 8 total, 8 up, 8 in 2024-04-02T04:50:18.019 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:17 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:18.019 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:17 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:50:18.019 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:17 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:50:18.019 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:17 smithi086 ceph-mon[24811]: pgmap v45: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:50:18.019 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:17 smithi086 ceph-mon[24811]: osdmap e62: 8 total, 8 up, 8 in 2024-04-02T04:50:18.019 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:17 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:18.019 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:17 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:50:18.019 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:17 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:50:18.109 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-mon[25697]: pgmap v45: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:50:18.109 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-mon[25697]: osdmap e62: 8 total, 8 up, 8 in 2024-04-02T04:50:18.109 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:18.109 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:50:18.109 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:50:18.110 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2024-04-02T04:50:18.110 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add encrypted dashboard json column" 2024-04-02T04:50:18.110 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-04-02T04:50:18.110 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2024-04-02T04:50:18.110 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2024-04-02T04:50:18.110 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2024-04-02T04:50:18.110 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2024-04-02T04:50:18.110 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-04-02T04:50:18.110 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2024-04-02T04:50:18.111 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2024-04-02T04:50:18.111 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2024-04-02T04:50:18.111 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2024-04-02T04:50:18.111 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2024-04-02T04:50:18.111 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2024-04-02T04:50:18.111 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2024-04-02T04:50:18.111 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2024-04-02T04:50:18.111 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2024-04-02T04:50:18.111 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2024-04-02T04:50:18.111 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2024-04-02T04:50:18.111 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2024-04-02T04:50:18.111 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2024-04-02T04:50:18.111 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2024-04-02T04:50:18.111 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2024-04-02T04:50:18.112 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column week_start in preferences" 2024-04-02T04:50:18.112 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2024-04-02T04:50:18.112 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2024-04-02T04:50:18.112 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2024-04-02T04:50:18.112 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2024-04-02T04:50:18.112 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2024-04-02T04:50:18.112 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-04-02T04:50:18.112 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-04-02T04:50:18.112 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-04-02T04:50:18.112 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v2" 2024-04-02T04:50:18.112 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-04-02T04:50:18.112 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="copy alert_rule_tag v1 to v2" 2024-04-02T04:50:18.112 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop table alert_rule_tag_v1" 2024-04-02T04:50:18.113 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2024-04-02T04:50:18.113 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2024-04-02T04:50:18.113 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2024-04-02T04:50:18.113 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2024-04-02T04:50:18.113 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2024-04-02T04:50:18.113 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2024-04-02T04:50:18.113 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2024-04-02T04:50:18.113 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2024-04-02T04:50:18.113 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2024-04-02T04:50:18.113 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2024-04-02T04:50:18.113 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2024-04-02T04:50:18.113 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2024-04-02T04:50:18.114 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-04-02T04:50:18.114 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2024-04-02T04:50:18.114 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2024-04-02T04:50:18.114 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2024-04-02T04:50:18.114 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2024-04-02T04:50:18.114 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2024-04-02T04:50:18.114 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2024-04-02T04:50:18.114 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2024-04-02T04:50:18.114 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2024-04-02T04:50:18.114 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2024-04-02T04:50:18.114 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2024-04-02T04:50:18.114 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2024-04-02T04:50:18.114 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2024-04-02T04:50:18.114 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2024-04-02T04:50:18.115 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2024-04-02T04:50:18.115 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2024-04-02T04:50:18.115 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2024-04-02T04:50:18.115 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2024-04-02T04:50:18.115 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2024-04-02T04:50:18.115 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2024-04-02T04:50:18.115 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2024-04-02T04:50:18.115 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2024-04-02T04:50:18.115 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2024-04-02T04:50:18.115 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-04-02T04:50:18.115 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-04-02T04:50:18.115 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2024-04-02T04:50:18.115 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-04-02T04:50:18.115 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2024-04-02T04:50:18.116 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2024-04-02T04:50:18.116 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2024-04-02T04:50:18.116 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2024-04-02T04:50:18.117 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2024-04-02T04:50:18.117 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2024-04-02T04:50:18.117 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2024-04-02T04:50:18.117 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2024-04-02T04:50:18.117 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2024-04-02T04:50:18.117 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2024-04-02T04:50:18.117 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2024-04-02T04:50:18.117 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2024-04-02T04:50:18.117 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2024-04-02T04:50:18.117 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-04-02T04:50:18.117 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-04-02T04:50:18.118 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2024-04-02T04:50:18.118 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2024-04-02T04:50:18.118 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2024-04-02T04:50:18.118 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2024-04-02T04:50:18.118 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2024-04-02T04:50:18.118 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2024-04-02T04:50:18.118 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-04-02T04:50:18.118 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2024-04-02T04:50:18.118 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2024-04-02T04:50:18.118 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2024-04-02T04:50:18.118 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2024-04-02T04:50:18.118 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2024-04-02T04:50:18.118 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2024-04-02T04:50:18.119 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:17+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2024-04-02T04:50:18.119 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2024-04-02T04:50:18.119 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2024-04-02T04:50:18.119 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2024-04-02T04:50:18.119 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2024-04-02T04:50:18.119 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2024-04-02T04:50:18.119 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2024-04-02T04:50:18.119 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2024-04-02T04:50:18.119 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2024-04-02T04:50:18.119 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2024-04-02T04:50:18.119 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2024-04-02T04:50:18.119 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2024-04-02T04:50:18.119 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2024-04-02T04:50:18.119 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2024-04-02T04:50:18.120 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2024-04-02T04:50:18.120 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2024-04-02T04:50:18.120 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2024-04-02T04:50:18.120 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2024-04-02T04:50:18.120 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2024-04-02T04:50:18.120 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2024-04-02T04:50:18.120 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2024-04-02T04:50:18.120 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2024-04-02T04:50:18.120 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-04-02T04:50:18.120 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2024-04-02T04:50:18.120 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2024-04-02T04:50:18.120 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2024-04-02T04:50:18.120 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2024-04-02T04:50:18.120 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2024-04-02T04:50:18.121 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-04-02T04:50:18.121 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2024-04-02T04:50:18.121 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2024-04-02T04:50:18.121 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2024-04-02T04:50:18.121 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2024-04-02T04:50:18.121 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2024-04-02T04:50:18.121 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2024-04-02T04:50:18.121 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2024-04-02T04:50:18.121 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2024-04-02T04:50:18.121 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2024-04-02T04:50:18.121 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2024-04-02T04:50:18.122 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2024-04-02T04:50:18.122 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2024-04-02T04:50:18.122 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2024-04-02T04:50:18.122 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2024-04-02T04:50:18.122 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2024-04-02T04:50:18.122 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2024-04-02T04:50:18.122 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2024-04-02T04:50:18.122 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2024-04-02T04:50:18.122 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2024-04-02T04:50:18.122 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2024-04-02T04:50:18.122 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2024-04-02T04:50:18.122 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2024-04-02T04:50:18.122 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2024-04-02T04:50:18.123 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2024-04-02T04:50:18.123 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2024-04-02T04:50:18.123 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2024-04-02T04:50:18.123 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2024-04-02T04:50:18.123 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2024-04-02T04:50:18.123 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2024-04-02T04:50:18.123 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2024-04-02T04:50:18.123 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-04-02T04:50:18.123 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-04-02T04:50:18.123 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2024-04-02T04:50:18.123 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2024-04-02T04:50:18.123 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2024-04-02T04:50:18.123 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-04-02T04:50:18.124 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2024-04-02T04:50:18.124 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2024-04-02T04:50:18.124 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-04-02T04:50:18.124 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2024-04-02T04:50:18.124 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2024-04-02T04:50:18.124 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2024-04-02T04:50:18.124 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-04-02T04:50:18.124 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2024-04-02T04:50:18.124 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2024-04-02T04:50:18.124 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2024-04-02T04:50:18.124 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2024-04-02T04:50:18.124 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-04-02T04:50:18.124 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2024-04-02T04:50:18.124 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2024-04-02T04:50:18.124 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2024-04-02T04:50:18.125 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2024-04-02T04:50:18.125 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2024-04-02T04:50:18.125 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-04-02T04:50:18.125 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2024-04-02T04:50:18.125 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2024-04-02T04:50:18.125 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-04-02T04:50:18.125 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2024-04-02T04:50:18.125 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-04-02T04:50:18.125 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-04-02T04:50:18.125 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2024-04-02T04:50:18.125 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2024-04-02T04:50:18.125 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2024-04-02T04:50:18.126 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2024-04-02T04:50:18.126 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2024-04-02T04:50:18.126 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2024-04-02T04:50:18.126 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-04-02T04:50:18.126 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2024-04-02T04:50:18.126 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2024-04-02T04:50:18.126 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2024-04-02T04:50:18.126 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2024-04-02T04:50:18.126 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2024-04-02T04:50:18.126 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2024-04-02T04:50:18.126 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2024-04-02T04:50:18.126 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2024-04-02T04:50:18.126 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2024-04-02T04:50:18.127 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2024-04-02T04:50:18.127 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2024-04-02T04:50:18.127 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2024-04-02T04:50:18.127 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2024-04-02T04:50:18.127 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2024-04-02T04:50:18.127 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2024-04-02T04:50:18.194 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T04:50:18.194 INFO:teuthology.orchestra.run.smithi086.stdout: "id": "df9facb0-7d4f-420d-b58e-f088c86a2389", 2024-04-02T04:50:18.194 INFO:teuthology.orchestra.run.smithi086.stdout: "name": "default", 2024-04-02T04:50:18.195 INFO:teuthology.orchestra.run.smithi086.stdout: "api_name": "default", 2024-04-02T04:50:18.195 INFO:teuthology.orchestra.run.smithi086.stdout: "is_master": "true", 2024-04-02T04:50:18.195 INFO:teuthology.orchestra.run.smithi086.stdout: "endpoints": [], 2024-04-02T04:50:18.195 INFO:teuthology.orchestra.run.smithi086.stdout: "hostnames": [], 2024-04-02T04:50:18.195 INFO:teuthology.orchestra.run.smithi086.stdout: "hostnames_s3website": [], 2024-04-02T04:50:18.195 INFO:teuthology.orchestra.run.smithi086.stdout: "master_zone": "", 2024-04-02T04:50:18.195 INFO:teuthology.orchestra.run.smithi086.stdout: "zones": [], 2024-04-02T04:50:18.195 INFO:teuthology.orchestra.run.smithi086.stdout: "placement_targets": [], 2024-04-02T04:50:18.195 INFO:teuthology.orchestra.run.smithi086.stdout: "default_placement": "", 2024-04-02T04:50:18.195 INFO:teuthology.orchestra.run.smithi086.stdout: "realm_id": "9953c8db-4704-4a82-b62f-70d6707827e1", 2024-04-02T04:50:18.195 INFO:teuthology.orchestra.run.smithi086.stdout: "sync_policy": { 2024-04-02T04:50:18.195 INFO:teuthology.orchestra.run.smithi086.stdout: "groups": [] 2024-04-02T04:50:18.195 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-04-02T04:50:18.196 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T04:50:18.428 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2024-04-02T04:50:18.428 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2024-04-02T04:50:18.428 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2024-04-02T04:50:18.429 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2024-04-02T04:50:18.429 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2024-04-02T04:50:18.429 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2024-04-02T04:50:18.429 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2024-04-02T04:50:18.429 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2024-04-02T04:50:18.429 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2024-04-02T04:50:18.429 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2024-04-02T04:50:18.429 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2024-04-02T04:50:18.429 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2024-04-02T04:50:18.429 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2024-04-02T04:50:18.429 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2024-04-02T04:50:18.429 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2024-04-02T04:50:18.429 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2024-04-02T04:50:18.429 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2024-04-02T04:50:18.430 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2024-04-02T04:50:18.430 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2024-04-02T04:50:18.430 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2024-04-02T04:50:18.430 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2024-04-02T04:50:18.430 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2024-04-02T04:50:18.430 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2024-04-02T04:50:18.430 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2024-04-02T04:50:18.430 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2024-04-02T04:50:18.430 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2024-04-02T04:50:18.430 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2024-04-02T04:50:18.430 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2024-04-02T04:50:18.430 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=462.916363ms 2024-04-02T04:50:18.430 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-04-02T04:50:18.430 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-04-02T04:50:18.431 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-04-02T04:50:18.431 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-04-02T04:50:18.431 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-04-02T04:50:18.431 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-04-02T04:50:18.431 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-04-02T04:50:18.431 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-04-02T04:50:18.431 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-04-02T04:50:18.431 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-04-02T04:50:18.431 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:50:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:50:18+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-04-02T04:50:18.803 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default' 2024-04-02T04:50:20.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:19 smithi192 ceph-mon[25697]: pgmap v47: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-02T04:50:20.237 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:19 smithi086 ceph-mon[29505]: pgmap v47: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-02T04:50:20.238 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:19 smithi086 ceph-mon[24811]: pgmap v47: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-02T04:50:20.238 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:50:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:50:20] "GET /metrics HTTP/1.1" 200 191484 "" "Prometheus/2.33.4" 2024-04-02T04:50:21.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:21 smithi086 ceph-mon[29505]: pgmap v48: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-02T04:50:21.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:21 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:21.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:21 smithi086 ceph-mon[24811]: pgmap v48: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-02T04:50:21.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:21 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:21.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:21 smithi192 ceph-mon[25697]: pgmap v48: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-02T04:50:21.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:21 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:22.047 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T04:50:22.047 INFO:teuthology.orchestra.run.smithi086.stdout: "id": "4656463d-bddd-4da0-95d5-f855e39882c7", 2024-04-02T04:50:22.047 INFO:teuthology.orchestra.run.smithi086.stdout: "name": "z", 2024-04-02T04:50:22.047 INFO:teuthology.orchestra.run.smithi086.stdout: "domain_root": "z.rgw.meta:root", 2024-04-02T04:50:22.047 INFO:teuthology.orchestra.run.smithi086.stdout: "control_pool": "z.rgw.control", 2024-04-02T04:50:22.047 INFO:teuthology.orchestra.run.smithi086.stdout: "gc_pool": "z.rgw.log:gc", 2024-04-02T04:50:22.047 INFO:teuthology.orchestra.run.smithi086.stdout: "lc_pool": "z.rgw.log:lc", 2024-04-02T04:50:22.048 INFO:teuthology.orchestra.run.smithi086.stdout: "log_pool": "z.rgw.log", 2024-04-02T04:50:22.048 INFO:teuthology.orchestra.run.smithi086.stdout: "intent_log_pool": "z.rgw.log:intent", 2024-04-02T04:50:22.048 INFO:teuthology.orchestra.run.smithi086.stdout: "usage_log_pool": "z.rgw.log:usage", 2024-04-02T04:50:22.048 INFO:teuthology.orchestra.run.smithi086.stdout: "roles_pool": "z.rgw.meta:roles", 2024-04-02T04:50:22.048 INFO:teuthology.orchestra.run.smithi086.stdout: "reshard_pool": "z.rgw.log:reshard", 2024-04-02T04:50:22.048 INFO:teuthology.orchestra.run.smithi086.stdout: "user_keys_pool": "z.rgw.meta:users.keys", 2024-04-02T04:50:22.048 INFO:teuthology.orchestra.run.smithi086.stdout: "user_email_pool": "z.rgw.meta:users.email", 2024-04-02T04:50:22.048 INFO:teuthology.orchestra.run.smithi086.stdout: "user_swift_pool": "z.rgw.meta:users.swift", 2024-04-02T04:50:22.048 INFO:teuthology.orchestra.run.smithi086.stdout: "user_uid_pool": "z.rgw.meta:users.uid", 2024-04-02T04:50:22.048 INFO:teuthology.orchestra.run.smithi086.stdout: "otp_pool": "z.rgw.otp", 2024-04-02T04:50:22.048 INFO:teuthology.orchestra.run.smithi086.stdout: "system_key": { 2024-04-02T04:50:22.048 INFO:teuthology.orchestra.run.smithi086.stdout: "access_key": "", 2024-04-02T04:50:22.048 INFO:teuthology.orchestra.run.smithi086.stdout: "secret_key": "" 2024-04-02T04:50:22.048 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T04:50:22.048 INFO:teuthology.orchestra.run.smithi086.stdout: "placement_pools": [ 2024-04-02T04:50:22.049 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-04-02T04:50:22.049 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "default-placement", 2024-04-02T04:50:22.049 INFO:teuthology.orchestra.run.smithi086.stdout: "val": { 2024-04-02T04:50:22.049 INFO:teuthology.orchestra.run.smithi086.stdout: "index_pool": "z.rgw.buckets.index", 2024-04-02T04:50:22.049 INFO:teuthology.orchestra.run.smithi086.stdout: "storage_classes": { 2024-04-02T04:50:22.049 INFO:teuthology.orchestra.run.smithi086.stdout: "STANDARD": { 2024-04-02T04:50:22.049 INFO:teuthology.orchestra.run.smithi086.stdout: "data_pool": "z.rgw.buckets.data" 2024-04-02T04:50:22.049 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-04-02T04:50:22.049 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T04:50:22.049 INFO:teuthology.orchestra.run.smithi086.stdout: "data_extra_pool": "z.rgw.buckets.non-ec", 2024-04-02T04:50:22.049 INFO:teuthology.orchestra.run.smithi086.stdout: "index_type": 0 2024-04-02T04:50:22.049 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-04-02T04:50:22.049 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-04-02T04:50:22.049 INFO:teuthology.orchestra.run.smithi086.stdout: ], 2024-04-02T04:50:22.049 INFO:teuthology.orchestra.run.smithi086.stdout: "realm_id": "9953c8db-4704-4a82-b62f-70d6707827e1", 2024-04-02T04:50:22.050 INFO:teuthology.orchestra.run.smithi086.stdout: "notif_pool": "z.rgw.log:notif" 2024-04-02T04:50:22.050 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T04:50:22.633 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'radosgw-admin period update --rgw-realm=r --commit' 2024-04-02T04:50:22.870 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:22 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:22.870 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:22 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:23.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:22 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:23.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:23 smithi086 ceph-mon[24811]: pgmap v49: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 262 B/s rd, 1.8 KiB/s wr, 2 op/s 2024-04-02T04:50:23.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:23 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:23.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:23 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:23.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:23 smithi086 ceph-mon[29505]: pgmap v49: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 262 B/s rd, 1.8 KiB/s wr, 2 op/s 2024-04-02T04:50:23.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:23 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:23.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:23 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:24.079 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:23 smithi192 ceph-mon[25697]: pgmap v49: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 262 B/s rd, 1.8 KiB/s wr, 2 op/s 2024-04-02T04:50:24.079 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:23 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:24.079 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:23 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:25.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:24 smithi086 ceph-mon[29505]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-02T04:50:25.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:24 smithi086 ceph-mon[29505]: Reconfiguring daemon alertmanager.a on smithi086 2024-04-02T04:50:25.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:24 smithi086 ceph-mon[24811]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-02T04:50:25.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:24 smithi086 ceph-mon[24811]: Reconfiguring daemon alertmanager.a on smithi086 2024-04-02T04:50:25.090 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:24 smithi192 ceph-mon[25697]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-02T04:50:25.090 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:24 smithi192 ceph-mon[25697]: Reconfiguring daemon alertmanager.a on smithi086 2024-04-02T04:50:25.428 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:50:25 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:50:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:50:26.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:25 smithi086 ceph-mon[24811]: pgmap v50: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 815 B/s rd, 1.8 KiB/s wr, 3 op/s 2024-04-02T04:50:26.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:25 smithi086 ceph-mon[29505]: pgmap v50: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 815 B/s rd, 1.8 KiB/s wr, 3 op/s 2024-04-02T04:50:26.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:25 smithi192 ceph-mon[25697]: pgmap v50: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 815 B/s rd, 1.8 KiB/s wr, 3 op/s 2024-04-02T04:50:27.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:26 smithi086 ceph-mon[24811]: osdmap e63: 8 total, 8 up, 8 in 2024-04-02T04:50:27.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:26 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/3505651192' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-02T04:50:27.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:26 smithi086 ceph-mon[24811]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-02T04:50:27.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:26 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:27.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:26 smithi086 ceph-mon[29505]: osdmap e63: 8 total, 8 up, 8 in 2024-04-02T04:50:27.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:26 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/3505651192' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-02T04:50:27.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:26 smithi086 ceph-mon[29505]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-02T04:50:27.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:26 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:27.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:26 smithi192 ceph-mon[25697]: osdmap e63: 8 total, 8 up, 8 in 2024-04-02T04:50:27.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:26 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/3505651192' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-02T04:50:27.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:26 smithi192 ceph-mon[25697]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-02T04:50:27.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:26 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:27.684 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:50:27 smithi086 systemd[1]: Stopping Ceph alertmanager.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T04:50:27.685 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:50:27 smithi086 bash[49401]: Error: no container with name or ID "ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager.a" found: no such container 2024-04-02T04:50:27.685 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:50:27 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[47190]: level=info ts=2024-04-02T04:50:27.661Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-04-02T04:50:27.957 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:27 smithi086 ceph-mon[29505]: pgmap v52: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 733 B/s rd, 1.6 KiB/s wr, 2 op/s 2024-04-02T04:50:27.957 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:27 smithi086 ceph-mon[29505]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-04-02T04:50:27.957 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:27 smithi086 ceph-mon[29505]: osdmap e64: 8 total, 8 up, 8 in 2024-04-02T04:50:27.957 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:50:27 smithi086 podman[49411]: 2024-04-02 04:50:27.682020062 +0000 UTC m=+0.100388746 container died b29bd562b181718d87bc5b34dc152e29ff7569f218c03d6ba99f7da7f3d25518 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T04:50:27.958 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:27 smithi086 ceph-mon[24811]: pgmap v52: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 733 B/s rd, 1.6 KiB/s wr, 2 op/s 2024-04-02T04:50:27.958 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:27 smithi086 ceph-mon[24811]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-04-02T04:50:27.958 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:27 smithi086 ceph-mon[24811]: osdmap e64: 8 total, 8 up, 8 in 2024-04-02T04:50:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:27 smithi192 ceph-mon[25697]: pgmap v52: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 733 B/s rd, 1.6 KiB/s wr, 2 op/s 2024-04-02T04:50:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:27 smithi192 ceph-mon[25697]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-04-02T04:50:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:27 smithi192 ceph-mon[25697]: osdmap e64: 8 total, 8 up, 8 in 2024-04-02T04:50:28.333 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:50:28 smithi086 podman[49411]: 2024-04-02 04:50:28.044867682 +0000 UTC m=+0.463236371 container cleanup b29bd562b181718d87bc5b34dc152e29ff7569f218c03d6ba99f7da7f3d25518 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T04:50:28.334 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:50:28 smithi086 bash[49411]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a 2024-04-02T04:50:28.334 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:50:28 smithi086 bash[49437]: Error: no container with name or ID "ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager.a" found: no such container 2024-04-02T04:50:28.646 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:50:28 smithi086 podman[49425]: 2024-04-02 04:50:28.337500984 +0000 UTC m=+0.651588208 container remove b29bd562b181718d87bc5b34dc152e29ff7569f218c03d6ba99f7da7f3d25518 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T04:50:28.646 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:50:28 smithi086 podman[49425]: 2024-04-02 04:50:28.421698492 +0000 UTC m=+0.735785706 volume remove 5698ea09c45db90a46e2dc4765a0503efc79537e694171ff31a7017ed3d677b4 2024-04-02T04:50:28.647 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:50:28 smithi086 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@alertmanager.a.service: Deactivated successfully. 2024-04-02T04:50:28.647 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:50:28 smithi086 systemd[1]: Stopped Ceph alertmanager.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T04:50:28.647 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:50:28 smithi086 systemd[1]: Starting Ceph alertmanager.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T04:50:29.001 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:28 smithi086 ceph-mon[29505]: osdmap e65: 8 total, 8 up, 8 in 2024-04-02T04:50:29.001 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:28 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/3505651192' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-02T04:50:29.001 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:28 smithi086 ceph-mon[29505]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-02T04:50:29.002 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:28 smithi086 ceph-mon[24811]: osdmap e65: 8 total, 8 up, 8 in 2024-04-02T04:50:29.002 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:28 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/3505651192' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-02T04:50:29.002 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:28 smithi086 ceph-mon[24811]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-02T04:50:29.002 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:50:28 smithi086 podman[49508]: 2024-04-02 04:50:28.546532397 +0000 UTC m=+0.017805165 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-04-02T04:50:29.002 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:50:28 smithi086 podman[49508]: 2024-04-02 04:50:28.647326861 +0000 UTC m=+0.118599672 volume create 911b09756c47529e737fc0641d1767d1ad2898bcfddd204fd20400492fb9c722 2024-04-02T04:50:29.002 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:50:28 smithi086 podman[49508]: 2024-04-02 04:50:28.764619708 +0000 UTC m=+0.235892483 container create 24a36455c11437e04d8501bf54dc2d6d40b59f667fdb776472db88cca9e88dbc (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T04:50:29.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:28 smithi192 ceph-mon[25697]: osdmap e65: 8 total, 8 up, 8 in 2024-04-02T04:50:29.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:28 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/3505651192' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-02T04:50:29.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:28 smithi192 ceph-mon[25697]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-02T04:50:29.584 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:50:29 smithi086 podman[49508]: 2024-04-02 04:50:29.127268503 +0000 UTC m=+0.598541268 container init 24a36455c11437e04d8501bf54dc2d6d40b59f667fdb776472db88cca9e88dbc (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T04:50:29.584 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:50:29 smithi086 podman[49508]: 2024-04-02 04:50:29.12986053 +0000 UTC m=+0.601133305 container start 24a36455c11437e04d8501bf54dc2d6d40b59f667fdb776472db88cca9e88dbc (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T04:50:29.584 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:50:29 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=info ts=2024-04-02T04:50:29.139Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-04-02T04:50:29.584 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:50:29 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=info ts=2024-04-02T04:50:29.139Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-04-02T04:50:29.584 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:50:29 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=info ts=2024-04-02T04:50:29.140Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.86 port=9094 2024-04-02T04:50:29.584 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:50:29 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=info ts=2024-04-02T04:50:29.142Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-02T04:50:29.584 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:50:29 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=info ts=2024-04-02T04:50:29.167Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-02T04:50:29.584 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:50:29 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=info ts=2024-04-02T04:50:29.167Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-02T04:50:29.584 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:50:29 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=info ts=2024-04-02T04:50:29.169Z caller=main.go:518 msg=Listening address=:9093 2024-04-02T04:50:29.584 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:50:29 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=info ts=2024-04-02T04:50:29.169Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-04-02T04:50:29.584 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:50:29 smithi086 bash[49508]: 24a36455c11437e04d8501bf54dc2d6d40b59f667fdb776472db88cca9e88dbc 2024-04-02T04:50:29.585 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:50:29 smithi086 systemd[1]: Started Ceph alertmanager.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T04:50:29.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:29 smithi192 ceph-mon[25697]: pgmap v55: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 4.7 KiB/s rd, 341 B/s wr, 4 op/s 2024-04-02T04:50:29.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:29 smithi192 ceph-mon[25697]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-04-02T04:50:29.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:29 smithi192 ceph-mon[25697]: osdmap e66: 8 total, 8 up, 8 in 2024-04-02T04:50:29.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:29 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:30.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:29 smithi086 ceph-mon[24811]: pgmap v55: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 4.7 KiB/s rd, 341 B/s wr, 4 op/s 2024-04-02T04:50:30.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:29 smithi086 ceph-mon[24811]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-04-02T04:50:30.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:29 smithi086 ceph-mon[24811]: osdmap e66: 8 total, 8 up, 8 in 2024-04-02T04:50:30.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:29 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:30.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:29 smithi086 ceph-mon[29505]: pgmap v55: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 4.7 KiB/s rd, 341 B/s wr, 4 op/s 2024-04-02T04:50:30.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:29 smithi086 ceph-mon[29505]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-04-02T04:50:30.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:29 smithi086 ceph-mon[29505]: osdmap e66: 8 total, 8 up, 8 in 2024-04-02T04:50:30.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:29 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:30.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:50:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:50:30] "GET /metrics HTTP/1.1" 200 194199 "" "Prometheus/2.33.4" 2024-04-02T04:50:31.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:30 smithi086 ceph-mon[24811]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-02T04:50:31.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:30 smithi086 ceph-mon[24811]: Reconfiguring daemon prometheus.a on smithi192 2024-04-02T04:50:31.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:30 smithi086 ceph-mon[24811]: osdmap e67: 8 total, 8 up, 8 in 2024-04-02T04:50:31.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:30 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/4066351514' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-02T04:50:31.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:30 smithi086 ceph-mon[24811]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-02T04:50:31.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:30 smithi086 ceph-mon[29505]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-02T04:50:31.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:30 smithi086 ceph-mon[29505]: Reconfiguring daemon prometheus.a on smithi192 2024-04-02T04:50:31.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:30 smithi086 ceph-mon[29505]: osdmap e67: 8 total, 8 up, 8 in 2024-04-02T04:50:31.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:30 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/4066351514' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-02T04:50:31.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:30 smithi086 ceph-mon[29505]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-02T04:50:31.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:30 smithi192 ceph-mon[25697]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-02T04:50:31.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:30 smithi192 ceph-mon[25697]: Reconfiguring daemon prometheus.a on smithi192 2024-04-02T04:50:31.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:30 smithi192 ceph-mon[25697]: osdmap e67: 8 total, 8 up, 8 in 2024-04-02T04:50:31.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:30 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/4066351514' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-02T04:50:31.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:30 smithi192 ceph-mon[25697]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-02T04:50:31.583 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:50:31 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=info ts=2024-04-02T04:50:31.143Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.0006725s 2024-04-02T04:50:31.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:31 smithi192 ceph-mon[25697]: pgmap v58: 129 pgs: 64 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 5.7 KiB/s rd, 5 op/s 2024-04-02T04:50:31.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:31 smithi192 ceph-mon[25697]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-04-02T04:50:31.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:31 smithi192 ceph-mon[25697]: osdmap e68: 8 total, 8 up, 8 in 2024-04-02T04:50:31.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:31 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/4066351514' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-02T04:50:31.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:31 smithi192 ceph-mon[25697]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-02T04:50:32.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:31 smithi086 ceph-mon[24811]: pgmap v58: 129 pgs: 64 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 5.7 KiB/s rd, 5 op/s 2024-04-02T04:50:32.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:31 smithi086 ceph-mon[24811]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-04-02T04:50:32.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:31 smithi086 ceph-mon[24811]: osdmap e68: 8 total, 8 up, 8 in 2024-04-02T04:50:32.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:31 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/4066351514' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-02T04:50:32.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:31 smithi086 ceph-mon[24811]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-02T04:50:32.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:31 smithi086 ceph-mon[29505]: pgmap v58: 129 pgs: 64 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 5.7 KiB/s rd, 5 op/s 2024-04-02T04:50:32.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:31 smithi086 ceph-mon[29505]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-04-02T04:50:32.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:31 smithi086 ceph-mon[29505]: osdmap e68: 8 total, 8 up, 8 in 2024-04-02T04:50:32.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:31 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/4066351514' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-02T04:50:32.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:31 smithi086 ceph-mon[29505]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-02T04:50:32.847 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T04:50:32.847 INFO:teuthology.orchestra.run.smithi086.stdout: "id": "055dd19f-7e48-4d65-8955-9ebb27e40e93", 2024-04-02T04:50:32.847 INFO:teuthology.orchestra.run.smithi086.stdout: "epoch": 1, 2024-04-02T04:50:32.848 INFO:teuthology.orchestra.run.smithi086.stdout: "predecessor_uuid": "ef79ad70-97de-4f81-8759-520eb84d60dd", 2024-04-02T04:50:32.848 INFO:teuthology.orchestra.run.smithi086.stdout: "sync_status": [], 2024-04-02T04:50:32.848 INFO:teuthology.orchestra.run.smithi086.stdout: "period_map": { 2024-04-02T04:50:32.848 INFO:teuthology.orchestra.run.smithi086.stdout: "id": "055dd19f-7e48-4d65-8955-9ebb27e40e93", 2024-04-02T04:50:32.848 INFO:teuthology.orchestra.run.smithi086.stdout: "zonegroups": [ 2024-04-02T04:50:32.848 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-04-02T04:50:32.848 INFO:teuthology.orchestra.run.smithi086.stdout: "id": "df9facb0-7d4f-420d-b58e-f088c86a2389", 2024-04-02T04:50:32.848 INFO:teuthology.orchestra.run.smithi086.stdout: "name": "default", 2024-04-02T04:50:32.848 INFO:teuthology.orchestra.run.smithi086.stdout: "api_name": "default", 2024-04-02T04:50:32.848 INFO:teuthology.orchestra.run.smithi086.stdout: "is_master": "true", 2024-04-02T04:50:32.848 INFO:teuthology.orchestra.run.smithi086.stdout: "endpoints": [], 2024-04-02T04:50:32.848 INFO:teuthology.orchestra.run.smithi086.stdout: "hostnames": [], 2024-04-02T04:50:32.848 INFO:teuthology.orchestra.run.smithi086.stdout: "hostnames_s3website": [], 2024-04-02T04:50:32.848 INFO:teuthology.orchestra.run.smithi086.stdout: "master_zone": "4656463d-bddd-4da0-95d5-f855e39882c7", 2024-04-02T04:50:32.848 INFO:teuthology.orchestra.run.smithi086.stdout: "zones": [ 2024-04-02T04:50:32.849 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-04-02T04:50:32.849 INFO:teuthology.orchestra.run.smithi086.stdout: "id": "4656463d-bddd-4da0-95d5-f855e39882c7", 2024-04-02T04:50:32.849 INFO:teuthology.orchestra.run.smithi086.stdout: "name": "z", 2024-04-02T04:50:32.849 INFO:teuthology.orchestra.run.smithi086.stdout: "endpoints": [], 2024-04-02T04:50:32.849 INFO:teuthology.orchestra.run.smithi086.stdout: "log_meta": "false", 2024-04-02T04:50:32.849 INFO:teuthology.orchestra.run.smithi086.stdout: "log_data": "false", 2024-04-02T04:50:32.849 INFO:teuthology.orchestra.run.smithi086.stdout: "bucket_index_max_shards": 11, 2024-04-02T04:50:32.849 INFO:teuthology.orchestra.run.smithi086.stdout: "read_only": "false", 2024-04-02T04:50:32.849 INFO:teuthology.orchestra.run.smithi086.stdout: "tier_type": "", 2024-04-02T04:50:32.849 INFO:teuthology.orchestra.run.smithi086.stdout: "sync_from_all": "true", 2024-04-02T04:50:32.849 INFO:teuthology.orchestra.run.smithi086.stdout: "sync_from": [], 2024-04-02T04:50:32.849 INFO:teuthology.orchestra.run.smithi086.stdout: "redirect_zone": "" 2024-04-02T04:50:32.849 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-04-02T04:50:32.849 INFO:teuthology.orchestra.run.smithi086.stdout: ], 2024-04-02T04:50:32.850 INFO:teuthology.orchestra.run.smithi086.stdout: "placement_targets": [ 2024-04-02T04:50:32.850 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-04-02T04:50:32.850 INFO:teuthology.orchestra.run.smithi086.stdout: "name": "default-placement", 2024-04-02T04:50:32.850 INFO:teuthology.orchestra.run.smithi086.stdout: "tags": [], 2024-04-02T04:50:32.850 INFO:teuthology.orchestra.run.smithi086.stdout: "storage_classes": [ 2024-04-02T04:50:32.850 INFO:teuthology.orchestra.run.smithi086.stdout: "STANDARD" 2024-04-02T04:50:32.850 INFO:teuthology.orchestra.run.smithi086.stdout: ] 2024-04-02T04:50:32.850 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-04-02T04:50:32.850 INFO:teuthology.orchestra.run.smithi086.stdout: ], 2024-04-02T04:50:32.850 INFO:teuthology.orchestra.run.smithi086.stdout: "default_placement": "default-placement", 2024-04-02T04:50:32.850 INFO:teuthology.orchestra.run.smithi086.stdout: "realm_id": "9953c8db-4704-4a82-b62f-70d6707827e1", 2024-04-02T04:50:32.850 INFO:teuthology.orchestra.run.smithi086.stdout: "sync_policy": { 2024-04-02T04:50:32.850 INFO:teuthology.orchestra.run.smithi086.stdout: "groups": [] 2024-04-02T04:50:32.850 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-04-02T04:50:32.851 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-04-02T04:50:32.851 INFO:teuthology.orchestra.run.smithi086.stdout: ], 2024-04-02T04:50:32.851 INFO:teuthology.orchestra.run.smithi086.stdout: "short_zone_ids": [ 2024-04-02T04:50:32.851 INFO:teuthology.orchestra.run.smithi086.stdout: { 2024-04-02T04:50:32.851 INFO:teuthology.orchestra.run.smithi086.stdout: "key": "4656463d-bddd-4da0-95d5-f855e39882c7", 2024-04-02T04:50:32.851 INFO:teuthology.orchestra.run.smithi086.stdout: "val": 2837460962 2024-04-02T04:50:32.851 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-04-02T04:50:32.851 INFO:teuthology.orchestra.run.smithi086.stdout: ] 2024-04-02T04:50:32.851 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T04:50:32.851 INFO:teuthology.orchestra.run.smithi086.stdout: "master_zonegroup": "df9facb0-7d4f-420d-b58e-f088c86a2389", 2024-04-02T04:50:32.851 INFO:teuthology.orchestra.run.smithi086.stdout: "master_zone": "4656463d-bddd-4da0-95d5-f855e39882c7", 2024-04-02T04:50:32.851 INFO:teuthology.orchestra.run.smithi086.stdout: "period_config": { 2024-04-02T04:50:32.851 INFO:teuthology.orchestra.run.smithi086.stdout: "bucket_quota": { 2024-04-02T04:50:32.851 INFO:teuthology.orchestra.run.smithi086.stdout: "enabled": false, 2024-04-02T04:50:32.851 INFO:teuthology.orchestra.run.smithi086.stdout: "check_on_raw": false, 2024-04-02T04:50:32.852 INFO:teuthology.orchestra.run.smithi086.stdout: "max_size": -1, 2024-04-02T04:50:32.852 INFO:teuthology.orchestra.run.smithi086.stdout: "max_size_kb": 0, 2024-04-02T04:50:32.852 INFO:teuthology.orchestra.run.smithi086.stdout: "max_objects": -1 2024-04-02T04:50:32.852 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T04:50:32.852 INFO:teuthology.orchestra.run.smithi086.stdout: "user_quota": { 2024-04-02T04:50:32.852 INFO:teuthology.orchestra.run.smithi086.stdout: "enabled": false, 2024-04-02T04:50:32.852 INFO:teuthology.orchestra.run.smithi086.stdout: "check_on_raw": false, 2024-04-02T04:50:32.852 INFO:teuthology.orchestra.run.smithi086.stdout: "max_size": -1, 2024-04-02T04:50:32.852 INFO:teuthology.orchestra.run.smithi086.stdout: "max_size_kb": 0, 2024-04-02T04:50:32.852 INFO:teuthology.orchestra.run.smithi086.stdout: "max_objects": -1 2024-04-02T04:50:32.852 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T04:50:32.852 INFO:teuthology.orchestra.run.smithi086.stdout: "user_ratelimit": { 2024-04-02T04:50:32.852 INFO:teuthology.orchestra.run.smithi086.stdout: "max_read_ops": 0, 2024-04-02T04:50:32.852 INFO:teuthology.orchestra.run.smithi086.stdout: "max_write_ops": 0, 2024-04-02T04:50:32.853 INFO:teuthology.orchestra.run.smithi086.stdout: "max_read_bytes": 0, 2024-04-02T04:50:32.853 INFO:teuthology.orchestra.run.smithi086.stdout: "max_write_bytes": 0, 2024-04-02T04:50:32.853 INFO:teuthology.orchestra.run.smithi086.stdout: "enabled": false 2024-04-02T04:50:32.853 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T04:50:32.853 INFO:teuthology.orchestra.run.smithi086.stdout: "bucket_ratelimit": { 2024-04-02T04:50:32.853 INFO:teuthology.orchestra.run.smithi086.stdout: "max_read_ops": 0, 2024-04-02T04:50:32.853 INFO:teuthology.orchestra.run.smithi086.stdout: "max_write_ops": 0, 2024-04-02T04:50:32.853 INFO:teuthology.orchestra.run.smithi086.stdout: "max_read_bytes": 0, 2024-04-02T04:50:32.853 INFO:teuthology.orchestra.run.smithi086.stdout: "max_write_bytes": 0, 2024-04-02T04:50:32.853 INFO:teuthology.orchestra.run.smithi086.stdout: "enabled": false 2024-04-02T04:50:32.853 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T04:50:32.853 INFO:teuthology.orchestra.run.smithi086.stdout: "anonymous_ratelimit": { 2024-04-02T04:50:32.853 INFO:teuthology.orchestra.run.smithi086.stdout: "max_read_ops": 0, 2024-04-02T04:50:32.853 INFO:teuthology.orchestra.run.smithi086.stdout: "max_write_ops": 0, 2024-04-02T04:50:32.853 INFO:teuthology.orchestra.run.smithi086.stdout: "max_read_bytes": 0, 2024-04-02T04:50:32.853 INFO:teuthology.orchestra.run.smithi086.stdout: "max_write_bytes": 0, 2024-04-02T04:50:32.854 INFO:teuthology.orchestra.run.smithi086.stdout: "enabled": false 2024-04-02T04:50:32.854 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-04-02T04:50:32.854 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T04:50:32.854 INFO:teuthology.orchestra.run.smithi086.stdout: "realm_id": "9953c8db-4704-4a82-b62f-70d6707827e1", 2024-04-02T04:50:32.854 INFO:teuthology.orchestra.run.smithi086.stdout: "realm_name": "r", 2024-04-02T04:50:32.854 INFO:teuthology.orchestra.run.smithi086.stdout: "realm_epoch": 2 2024-04-02T04:50:32.854 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T04:50:33.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:32 smithi086 ceph-mon[24811]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-04-02T04:50:33.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:32 smithi086 ceph-mon[24811]: osdmap e69: 8 total, 8 up, 8 in 2024-04-02T04:50:33.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:32 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/4066351514' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-02T04:50:33.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:32 smithi086 ceph-mon[24811]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-02T04:50:33.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:32 smithi086 ceph-mon[29505]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-04-02T04:50:33.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:32 smithi086 ceph-mon[29505]: osdmap e69: 8 total, 8 up, 8 in 2024-04-02T04:50:33.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:32 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/4066351514' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-02T04:50:33.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:32 smithi086 ceph-mon[29505]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-02T04:50:33.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:32 smithi192 ceph-mon[25697]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-04-02T04:50:33.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:32 smithi192 ceph-mon[25697]: osdmap e69: 8 total, 8 up, 8 in 2024-04-02T04:50:33.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:32 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/4066351514' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-02T04:50:33.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:32 smithi192 ceph-mon[25697]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-02T04:50:33.179 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:32 smithi192 systemd[1]: Stopping Ceph prometheus.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T04:50:33.179 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:32 smithi192 bash[42483]: Error: no container with name or ID "ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus.a" found: no such container 2024-04-02T04:50:33.179 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[40817]: ts=2024-04-02T04:50:32.978Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-02T04:50:33.179 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[40817]: ts=2024-04-02T04:50:32.978Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-02T04:50:33.179 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[40817]: ts=2024-04-02T04:50:32.978Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-02T04:50:33.179 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[40817]: ts=2024-04-02T04:50:32.978Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-02T04:50:33.179 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[40817]: ts=2024-04-02T04:50:32.978Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-02T04:50:33.179 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[40817]: ts=2024-04-02T04:50:32.978Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-02T04:50:33.179 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[40817]: ts=2024-04-02T04:50:32.978Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-02T04:50:33.179 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[40817]: ts=2024-04-02T04:50:32.978Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-02T04:50:33.179 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[40817]: ts=2024-04-02T04:50:32.978Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-02T04:50:33.179 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[40817]: ts=2024-04-02T04:50:32.979Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-02T04:50:33.179 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[40817]: ts=2024-04-02T04:50:32.979Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-02T04:50:33.179 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[40817]: ts=2024-04-02T04:50:32.979Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-02T04:50:33.180 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:32 smithi192 podman[42494]: 2024-04-02 04:50:32.988671004 +0000 UTC m=+0.114377690 container died 93d642a7521e1350074c2ee753bfddffcb1b9f5299b96b339667bc83a0d84f6f (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T04:50:33.491 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000' 2024-04-02T04:50:33.678 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:33 smithi192 podman[42494]: 2024-04-02 04:50:33.319297071 +0000 UTC m=+0.445003765 container cleanup 93d642a7521e1350074c2ee753bfddffcb1b9f5299b96b339667bc83a0d84f6f (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T04:50:33.678 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:33 smithi192 bash[42494]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a 2024-04-02T04:50:33.678 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:33 smithi192 bash[42520]: Error: no container with name or ID "ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus.a" found: no such container 2024-04-02T04:50:33.678 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:33 smithi192 podman[42508]: 2024-04-02 04:50:33.537153217 +0000 UTC m=+0.551399873 container remove 93d642a7521e1350074c2ee753bfddffcb1b9f5299b96b339667bc83a0d84f6f (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T04:50:33.678 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:33 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@prometheus.a.service: Deactivated successfully. 2024-04-02T04:50:33.678 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:33 smithi192 systemd[1]: Stopped Ceph prometheus.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T04:50:33.678 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:33 smithi192 systemd[1]: Starting Ceph prometheus.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T04:50:34.031 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:33 smithi192 ceph-mon[25697]: pgmap v61: 129 pgs: 4 unknown, 125 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-04-02T04:50:34.031 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:33 smithi192 ceph-mon[25697]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-04-02T04:50:34.031 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:33 smithi192 ceph-mon[25697]: osdmap e70: 8 total, 8 up, 8 in 2024-04-02T04:50:34.032 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:33 smithi192 podman[42591]: 2024-04-02 04:50:33.666398854 +0000 UTC m=+0.018861363 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-02T04:50:34.032 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:33 smithi192 podman[42591]: 2024-04-02 04:50:33.788281864 +0000 UTC m=+0.140744354 container create 782ae1d5e7f93611fd1424153fd849cd9e64b981a84f452dd70541a64d702187 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T04:50:34.032 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:33 smithi192 podman[42591]: 2024-04-02 04:50:33.949166035 +0000 UTC m=+0.301628540 container init 782ae1d5e7f93611fd1424153fd849cd9e64b981a84f452dd70541a64d702187 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T04:50:34.032 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:33 smithi192 podman[42591]: 2024-04-02 04:50:33.951767377 +0000 UTC m=+0.304229867 container start 782ae1d5e7f93611fd1424153fd849cd9e64b981a84f452dd70541a64d702187 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T04:50:34.032 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:33 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[42606]: ts=2024-04-02T04:50:33.974Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-02T04:50:34.032 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:33 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[42606]: ts=2024-04-02T04:50:33.974Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-02T04:50:34.032 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:33 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[42606]: ts=2024-04-02T04:50:33.974Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-02T04:50:34.032 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:33 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[42606]: ts=2024-04-02T04:50:33.974Z caller=main.go:518 level=info host_details="(Linux 5.14.0-434.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Mar 26 19:20:47 UTC 2024 x86_64 smithi192 (none))" 2024-04-02T04:50:34.032 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:33 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[42606]: ts=2024-04-02T04:50:33.974Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-02T04:50:34.032 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:33 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[42606]: ts=2024-04-02T04:50:33.974Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-02T04:50:34.033 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:33 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[42606]: ts=2024-04-02T04:50:33.975Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-02T04:50:34.033 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:33 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[42606]: ts=2024-04-02T04:50:33.976Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-02T04:50:34.033 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:33 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[42606]: ts=2024-04-02T04:50:33.976Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-02T04:50:34.033 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:33 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[42606]: ts=2024-04-02T04:50:33.978Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-02T04:50:34.033 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:33 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[42606]: ts=2024-04-02T04:50:33.978Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.548µs 2024-04-02T04:50:34.033 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:33 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[42606]: ts=2024-04-02T04:50:33.978Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-02T04:50:34.033 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:34 smithi192 bash[42591]: 782ae1d5e7f93611fd1424153fd849cd9e64b981a84f452dd70541a64d702187 2024-04-02T04:50:34.033 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:34 smithi192 systemd[1]: Started Ceph prometheus.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T04:50:34.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:33 smithi086 ceph-mon[29505]: pgmap v61: 129 pgs: 4 unknown, 125 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-04-02T04:50:34.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:33 smithi086 ceph-mon[29505]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-04-02T04:50:34.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:33 smithi086 ceph-mon[29505]: osdmap e70: 8 total, 8 up, 8 in 2024-04-02T04:50:34.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:33 smithi086 ceph-mon[24811]: pgmap v61: 129 pgs: 4 unknown, 125 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-04-02T04:50:34.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:33 smithi086 ceph-mon[24811]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-04-02T04:50:34.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:33 smithi086 ceph-mon[24811]: osdmap e70: 8 total, 8 up, 8 in 2024-04-02T04:50:34.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:34 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[42606]: ts=2024-04-02T04:50:34.193Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-04-02T04:50:34.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:34 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[42606]: ts=2024-04-02T04:50:34.193Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-04-02T04:50:34.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:34 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[42606]: ts=2024-04-02T04:50:34.193Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=22.174µs wal_replay_duration=215.408215ms total_replay_duration=215.44305ms 2024-04-02T04:50:34.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:34 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[42606]: ts=2024-04-02T04:50:34.195Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-02T04:50:34.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:34 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[42606]: ts=2024-04-02T04:50:34.195Z caller=main.go:947 level=info msg="TSDB started" 2024-04-02T04:50:34.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:34 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[42606]: ts=2024-04-02T04:50:34.195Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-02T04:50:34.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:34 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[42606]: ts=2024-04-02T04:50:34.210Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.532674ms db_storage=603ns remote_storage=1.573µs web_handler=431ns query_engine=974ns scrape=360.063µs scrape_sd=24.894µs notify=27.499µs notify_sd=8.222µs rules=13.884323ms 2024-04-02T04:50:34.429 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:50:34 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[42606]: ts=2024-04-02T04:50:34.210Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-02T04:50:35.152 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:35.152 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T04:50:35.152 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[29505]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T04:50:35.152 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.86:9093"}]: dispatch 2024-04-02T04:50:35.152 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[29505]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.86:9093"}]: dispatch 2024-04-02T04:50:35.152 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:35.152 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T04:50:35.152 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[29505]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T04:50:35.153 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.192:9095"}]: dispatch 2024-04-02T04:50:35.153 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[29505]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.192:9095"}]: dispatch 2024-04-02T04:50:35.153 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:35.153 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T04:50:35.153 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[29505]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T04:50:35.153 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.192:3000"}]: dispatch 2024-04-02T04:50:35.153 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[29505]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.192:3000"}]: dispatch 2024-04-02T04:50:35.153 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:35.153 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:50:35.153 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:50:35.153 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[29505]: pgmap v63: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 208 B/s rd, 416 B/s wr, 1 op/s 2024-04-02T04:50:35.153 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:35.154 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T04:50:35.154 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[24811]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T04:50:35.154 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.86:9093"}]: dispatch 2024-04-02T04:50:35.154 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[24811]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.86:9093"}]: dispatch 2024-04-02T04:50:35.154 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:35.154 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T04:50:35.154 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[24811]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T04:50:35.154 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.192:9095"}]: dispatch 2024-04-02T04:50:35.154 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[24811]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.192:9095"}]: dispatch 2024-04-02T04:50:35.154 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:35.154 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T04:50:35.154 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[24811]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T04:50:35.154 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.192:3000"}]: dispatch 2024-04-02T04:50:35.155 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[24811]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.192:3000"}]: dispatch 2024-04-02T04:50:35.155 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:35.155 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:50:35.155 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:50:35.155 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:35 smithi086 ceph-mon[24811]: pgmap v63: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 208 B/s rd, 416 B/s wr, 1 op/s 2024-04-02T04:50:35.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:35 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:35.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:35 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T04:50:35.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:35 smithi192 ceph-mon[25697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T04:50:35.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:35 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.86:9093"}]: dispatch 2024-04-02T04:50:35.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:35 smithi192 ceph-mon[25697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.86:9093"}]: dispatch 2024-04-02T04:50:35.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:35 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:35.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:35 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T04:50:35.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:35 smithi192 ceph-mon[25697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T04:50:35.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:35 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.192:9095"}]: dispatch 2024-04-02T04:50:35.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:35 smithi192 ceph-mon[25697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.192:9095"}]: dispatch 2024-04-02T04:50:35.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:35 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:35.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:35 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T04:50:35.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:35 smithi192 ceph-mon[25697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T04:50:35.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:35 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.192:3000"}]: dispatch 2024-04-02T04:50:35.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:35 smithi192 ceph-mon[25697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.192:3000"}]: dispatch 2024-04-02T04:50:35.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:35 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:35.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:35 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:50:35.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:35 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:50:35.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:35 smithi192 ceph-mon[25697]: pgmap v63: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 208 B/s rd, 416 B/s wr, 1 op/s 2024-04-02T04:50:35.563 INFO:teuthology.orchestra.run.smithi086.stdout:Scheduled rgw.foo update... 2024-04-02T04:50:36.582 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph osd pool create foo' 2024-04-02T04:50:36.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:36 smithi086 ceph-mon[29505]: from='client.24574 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:50:36.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:36 smithi086 ceph-mon[29505]: Saving service rgw.foo spec with placement count:2 2024-04-02T04:50:36.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:36 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:36.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:36 smithi086 ceph-mon[24811]: from='client.24574 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:50:36.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:36 smithi086 ceph-mon[24811]: Saving service rgw.foo spec with placement count:2 2024-04-02T04:50:36.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:36 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:36.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:36 smithi192 ceph-mon[25697]: from='client.24574 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:50:36.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:36 smithi192 ceph-mon[25697]: Saving service rgw.foo spec with placement count:2 2024-04-02T04:50:36.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:36 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:37.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:37 smithi192 ceph-mon[25697]: pgmap v64: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-04-02T04:50:37.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:37 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:37.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:37 smithi086 ceph-mon[29505]: pgmap v64: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-04-02T04:50:37.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:37 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:37.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:37 smithi086 ceph-mon[24811]: pgmap v64: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-04-02T04:50:37.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:37 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:39.164 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:50:39 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=info ts=2024-04-02T04:50:39.144Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002195742s 2024-04-02T04:50:39.745 INFO:teuthology.orchestra.run.smithi086.stderr:pool 'foo' created 2024-04-02T04:50:39.813 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:39 smithi086 ceph-mon[29505]: pgmap v65: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.1 KiB/s wr, 3 op/s 2024-04-02T04:50:39.813 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:39 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1530622311' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-02T04:50:39.813 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:39 smithi086 ceph-mon[29505]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-02T04:50:39.813 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:39 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:39.813 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:39 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:39.813 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:39 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:39.814 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:39 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:39.814 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:39 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:39.814 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:39 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi086.bxzxlp", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T04:50:39.814 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:39 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi086.bxzxlp", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-02T04:50:39.814 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:39 smithi086 ceph-mon[24811]: pgmap v65: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.1 KiB/s wr, 3 op/s 2024-04-02T04:50:39.814 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:39 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1530622311' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-02T04:50:39.814 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:39 smithi086 ceph-mon[24811]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-02T04:50:39.814 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:39 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:39.814 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:39 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:39.814 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:39 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:39.814 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:39 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:39.815 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:39 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:39.815 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:39 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi086.bxzxlp", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T04:50:39.815 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:39 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi086.bxzxlp", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-02T04:50:40.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:39 smithi192 ceph-mon[25697]: pgmap v65: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.1 KiB/s wr, 3 op/s 2024-04-02T04:50:40.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:39 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1530622311' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-02T04:50:40.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:39 smithi192 ceph-mon[25697]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-02T04:50:40.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:39 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:40.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:39 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:40.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:39 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:40.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:39 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:40.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:39 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:40.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:39 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi086.bxzxlp", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T04:50:40.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:39 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi086.bxzxlp", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-02T04:50:40.445 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:50:40 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:50:40] "GET /metrics HTTP/1.1" 200 194199 "" "Prometheus/2.33.4" 2024-04-02T04:50:40.478 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'rbd pool init foo' 2024-04-02T04:50:41.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:40 smithi086 ceph-mon[24811]: Saving service rgw.foo spec with placement count:2 2024-04-02T04:50:41.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:40 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:41.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:40 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:50:41.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:40 smithi086 ceph-mon[24811]: Deploying daemon rgw.foo.smithi086.bxzxlp on smithi086 2024-04-02T04:50:41.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:40 smithi086 ceph-mon[24811]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-04-02T04:50:41.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:40 smithi086 ceph-mon[24811]: osdmap e71: 8 total, 8 up, 8 in 2024-04-02T04:50:41.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:40 smithi086 ceph-mon[29505]: Saving service rgw.foo spec with placement count:2 2024-04-02T04:50:41.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:40 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:41.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:40 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:50:41.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:40 smithi086 ceph-mon[29505]: Deploying daemon rgw.foo.smithi086.bxzxlp on smithi086 2024-04-02T04:50:41.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:40 smithi086 ceph-mon[29505]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-04-02T04:50:41.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:40 smithi086 ceph-mon[29505]: osdmap e71: 8 total, 8 up, 8 in 2024-04-02T04:50:41.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:40 smithi192 ceph-mon[25697]: Saving service rgw.foo spec with placement count:2 2024-04-02T04:50:41.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:40 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:41.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:40 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:50:41.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:40 smithi192 ceph-mon[25697]: Deploying daemon rgw.foo.smithi086.bxzxlp on smithi086 2024-04-02T04:50:41.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:40 smithi192 ceph-mon[25697]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-04-02T04:50:41.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:40 smithi192 ceph-mon[25697]: osdmap e71: 8 total, 8 up, 8 in 2024-04-02T04:50:41.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:41 smithi086 ceph-mon[24811]: pgmap v67: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 895 B/s wr, 2 op/s 2024-04-02T04:50:41.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:41 smithi086 ceph-mon[24811]: osdmap e72: 8 total, 8 up, 8 in 2024-04-02T04:50:41.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:41 smithi086 ceph-mon[29505]: pgmap v67: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 895 B/s wr, 2 op/s 2024-04-02T04:50:41.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:41 smithi086 ceph-mon[29505]: osdmap e72: 8 total, 8 up, 8 in 2024-04-02T04:50:42.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:41 smithi192 ceph-mon[25697]: pgmap v67: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 895 B/s wr, 2 op/s 2024-04-02T04:50:42.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:41 smithi192 ceph-mon[25697]: osdmap e72: 8 total, 8 up, 8 in 2024-04-02T04:50:42.925 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:42 smithi086 ceph-mon[24811]: osdmap e73: 8 total, 8 up, 8 in 2024-04-02T04:50:42.925 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:42 smithi086 ceph-mon[29505]: osdmap e73: 8 total, 8 up, 8 in 2024-04-02T04:50:43.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:42 smithi192 ceph-mon[25697]: osdmap e73: 8 total, 8 up, 8 in 2024-04-02T04:50:44.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:43 smithi086 ceph-mon[24811]: pgmap v70: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.2 KiB/s wr, 2 op/s 2024-04-02T04:50:44.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:43 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/3608128193' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-02T04:50:44.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:43 smithi086 ceph-mon[29505]: pgmap v70: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.2 KiB/s wr, 2 op/s 2024-04-02T04:50:44.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:43 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/3608128193' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-02T04:50:44.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:43 smithi192 ceph-mon[25697]: pgmap v70: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.2 KiB/s wr, 2 op/s 2024-04-02T04:50:44.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:43 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/3608128193' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-02T04:50:44.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:44 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/3608128193' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-04-02T04:50:44.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:44 smithi192 ceph-mon[25697]: osdmap e74: 8 total, 8 up, 8 in 2024-04-02T04:50:44.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:44 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:44.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:44 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi192.pmlbot", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T04:50:44.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:44 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi192.pmlbot", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-02T04:50:44.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:44 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:44.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:44 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:50:44.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:44 smithi192 ceph-mon[25697]: Deploying daemon rgw.foo.smithi192.pmlbot on smithi192 2024-04-02T04:50:45.015 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:44 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/3608128193' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-04-02T04:50:45.015 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:44 smithi086 ceph-mon[24811]: osdmap e74: 8 total, 8 up, 8 in 2024-04-02T04:50:45.015 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:44 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:45.015 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:44 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi192.pmlbot", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T04:50:45.015 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:44 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi192.pmlbot", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-02T04:50:45.015 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:44 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:45.015 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:44 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:50:45.016 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:44 smithi086 ceph-mon[24811]: Deploying daemon rgw.foo.smithi192.pmlbot on smithi192 2024-04-02T04:50:45.016 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:44 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/3608128193' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-04-02T04:50:45.016 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:44 smithi086 ceph-mon[29505]: osdmap e74: 8 total, 8 up, 8 in 2024-04-02T04:50:45.016 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:44 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:45.016 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:44 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi192.pmlbot", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T04:50:45.016 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:44 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi192.pmlbot", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-02T04:50:45.016 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:44 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:45.016 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:44 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:50:45.016 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:44 smithi086 ceph-mon[29505]: Deploying daemon rgw.foo.smithi192.pmlbot on smithi192 2024-04-02T04:50:45.584 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:50:45 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:50:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:50:45.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:45 smithi192 ceph-mon[25697]: pgmap v72: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:50:45.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:45 smithi192 ceph-mon[25697]: osdmap e75: 8 total, 8 up, 8 in 2024-04-02T04:50:46.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:45 smithi086 ceph-mon[24811]: pgmap v72: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:50:46.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:45 smithi086 ceph-mon[24811]: osdmap e75: 8 total, 8 up, 8 in 2024-04-02T04:50:46.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:45 smithi086 ceph-mon[29505]: pgmap v72: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:50:46.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:45 smithi086 ceph-mon[29505]: osdmap e75: 8 total, 8 up, 8 in 2024-04-02T04:50:46.511 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch apply iscsi foo u p' 2024-04-02T04:50:46.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:46 smithi192 ceph-mon[25697]: osdmap e76: 8 total, 8 up, 8 in 2024-04-02T04:50:46.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:46 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:46.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:46 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:50:46.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:46 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:50:46.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:46 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:47.070 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:46 smithi086 ceph-mon[29505]: osdmap e76: 8 total, 8 up, 8 in 2024-04-02T04:50:47.070 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:46 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:47.071 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:46 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:50:47.071 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:46 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:50:47.071 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:46 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:47.071 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:46 smithi086 ceph-mon[24811]: osdmap e76: 8 total, 8 up, 8 in 2024-04-02T04:50:47.071 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:46 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:47.071 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:46 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:50:47.071 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:46 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:50:47.071 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:46 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:47.952 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:47 smithi086 ceph-mon[24811]: pgmap v75: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:50:47.952 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:47 smithi086 ceph-mon[29505]: pgmap v75: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:50:48.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:47 smithi192 ceph-mon[25697]: pgmap v75: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:50:48.365 INFO:teuthology.orchestra.run.smithi086.stdout:Scheduled iscsi.foo update... 2024-04-02T04:50:49.397 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:49 smithi086 ceph-mon[29505]: from='client.14727 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:50:49.397 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:49 smithi086 ceph-mon[29505]: Saving service iscsi.foo spec with placement count:1 2024-04-02T04:50:49.397 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:49 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:49.397 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:49 smithi086 ceph-mon[29505]: pgmap v76: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 251 KiB/s rd, 6.0 KiB/s wr, 460 op/s 2024-04-02T04:50:49.397 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:49 smithi086 ceph-mon[24811]: from='client.14727 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:50:49.397 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:49 smithi086 ceph-mon[24811]: Saving service iscsi.foo spec with placement count:1 2024-04-02T04:50:49.397 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:49 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:49.398 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:49 smithi086 ceph-mon[24811]: pgmap v76: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 251 KiB/s rd, 6.0 KiB/s wr, 460 op/s 2024-04-02T04:50:49.506 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'sleep 180' 2024-04-02T04:50:49.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:49 smithi192 ceph-mon[25697]: from='client.14727 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:50:49.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:49 smithi192 ceph-mon[25697]: Saving service iscsi.foo spec with placement count:1 2024-04-02T04:50:49.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:49 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:49.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:49 smithi192 ceph-mon[25697]: pgmap v76: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 251 KiB/s rd, 6.0 KiB/s wr, 460 op/s 2024-04-02T04:50:50.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:50:50 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:50:50] "GET /metrics HTTP/1.1" 200 197867 "" "Prometheus/2.33.4" 2024-04-02T04:50:51.073 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:50 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:51.073 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:50 smithi086 ceph-mon[29505]: osdmap e77: 8 total, 8 up, 8 in 2024-04-02T04:50:51.073 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:50 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:51.074 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:50 smithi086 ceph-mon[24811]: osdmap e77: 8 total, 8 up, 8 in 2024-04-02T04:50:51.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:50 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:51.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:50 smithi192 ceph-mon[25697]: osdmap e77: 8 total, 8 up, 8 in 2024-04-02T04:50:51.975 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:51 smithi086 ceph-mon[24811]: pgmap v78: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 251 KiB/s rd, 6.0 KiB/s wr, 460 op/s 2024-04-02T04:50:51.976 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:51 smithi086 ceph-mon[29505]: pgmap v78: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 251 KiB/s rd, 6.0 KiB/s wr, 460 op/s 2024-04-02T04:50:52.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:51 smithi192 ceph-mon[25697]: pgmap v78: 161 pgs: 161 active+clean; 584 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 251 KiB/s rd, 6.0 KiB/s wr, 460 op/s 2024-04-02T04:50:53.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:52 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:53.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:52 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:53.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:52 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:54.078 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:53 smithi192 ceph-mon[25697]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 234 KiB/s rd, 4.5 KiB/s wr, 414 op/s 2024-04-02T04:50:54.079 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:53 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:54.079 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:53 smithi192 ceph-mon[25697]: Checking dashboard <-> RGW credentials 2024-04-02T04:50:54.079 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:53 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:54.079 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:53 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:54.079 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:53 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:54.079 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:53 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T04:50:54.079 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:53 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-04-02T04:50:54.079 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:53 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:50:54.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:53 smithi086 ceph-mon[24811]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 234 KiB/s rd, 4.5 KiB/s wr, 414 op/s 2024-04-02T04:50:54.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:53 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:54.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:53 smithi086 ceph-mon[24811]: Checking dashboard <-> RGW credentials 2024-04-02T04:50:54.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:53 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:54.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:53 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:54.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:53 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:54.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:53 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T04:50:54.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:53 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-04-02T04:50:54.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:53 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:50:54.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:53 smithi086 ceph-mon[29505]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 234 KiB/s rd, 4.5 KiB/s wr, 414 op/s 2024-04-02T04:50:54.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:53 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:54.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:53 smithi086 ceph-mon[29505]: Checking dashboard <-> RGW credentials 2024-04-02T04:50:54.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:53 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:54.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:53 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:54.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:53 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:54.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:53 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T04:50:54.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:53 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-04-02T04:50:54.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:53 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:50:55.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:54 smithi086 ceph-mon[29505]: Checking pool "foo" exists for service iscsi.foo 2024-04-02T04:50:55.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:54 smithi086 ceph-mon[29505]: Deploying daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T04:50:55.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:54 smithi086 ceph-mon[29505]: mgrmap e20: y(active, since 118s), standbys: x 2024-04-02T04:50:55.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:54 smithi086 ceph-mon[24811]: Checking pool "foo" exists for service iscsi.foo 2024-04-02T04:50:55.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:54 smithi086 ceph-mon[24811]: Deploying daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T04:50:55.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:54 smithi086 ceph-mon[24811]: mgrmap e20: y(active, since 118s), standbys: x 2024-04-02T04:50:55.090 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:54 smithi192 ceph-mon[25697]: Checking pool "foo" exists for service iscsi.foo 2024-04-02T04:50:55.090 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:54 smithi192 ceph-mon[25697]: Deploying daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T04:50:55.090 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:54 smithi192 ceph-mon[25697]: mgrmap e20: y(active, since 118s), standbys: x 2024-04-02T04:50:55.430 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:50:55 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:50:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:50:56.027 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:55 smithi086 ceph-mon[29505]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 208 KiB/s rd, 4.0 KiB/s wr, 367 op/s 2024-04-02T04:50:56.027 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:55 smithi086 ceph-mon[24811]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 208 KiB/s rd, 4.0 KiB/s wr, 367 op/s 2024-04-02T04:50:56.028 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:55 smithi192 ceph-mon[25697]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 208 KiB/s rd, 4.0 KiB/s wr, 367 op/s 2024-04-02T04:50:57.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:56 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:57.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:56 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:50:57.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:56 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:50:57.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:56 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/994224851' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T04:50:57.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:56 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [7, 4]}]: dispatch 2024-04-02T04:50:57.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:56 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-04-02T04:50:57.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:56 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-04-02T04:50:57.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:56 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2024-04-02T04:50:57.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:56 smithi086 ceph-mon[24811]: pgmap v81: 161 pgs: 161 active+clean; 584 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 185 KiB/s rd, 3.6 KiB/s wr, 327 op/s 2024-04-02T04:50:57.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:56 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/695795728' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/368218600"}]: dispatch 2024-04-02T04:50:57.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:56 smithi086 ceph-mon[24811]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/368218600"}]: dispatch 2024-04-02T04:50:57.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:56 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:50:57.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:56 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:50:57.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:56 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:57.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:56 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:57.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:56 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:50:57.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:56 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:50:57.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:56 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/994224851' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T04:50:57.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:56 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [7, 4]}]: dispatch 2024-04-02T04:50:57.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:56 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-04-02T04:50:57.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:56 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-04-02T04:50:57.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:56 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2024-04-02T04:50:57.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:56 smithi086 ceph-mon[29505]: pgmap v81: 161 pgs: 161 active+clean; 584 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 185 KiB/s rd, 3.6 KiB/s wr, 327 op/s 2024-04-02T04:50:57.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:56 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/695795728' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/368218600"}]: dispatch 2024-04-02T04:50:57.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:56 smithi086 ceph-mon[29505]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/368218600"}]: dispatch 2024-04-02T04:50:57.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:56 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:50:57.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:56 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:50:57.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:56 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:57.372 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:56 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:57.372 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:56 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:50:57.373 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:56 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:50:57.373 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:56 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/994224851' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T04:50:57.373 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:56 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [7, 4]}]: dispatch 2024-04-02T04:50:57.373 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:56 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-04-02T04:50:57.373 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:56 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-04-02T04:50:57.373 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:56 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2024-04-02T04:50:57.373 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:56 smithi192 ceph-mon[25697]: pgmap v81: 161 pgs: 161 active+clean; 584 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 185 KiB/s rd, 3.6 KiB/s wr, 327 op/s 2024-04-02T04:50:57.373 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:56 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/695795728' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/368218600"}]: dispatch 2024-04-02T04:50:57.373 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:56 smithi192 ceph-mon[25697]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/368218600"}]: dispatch 2024-04-02T04:50:57.373 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:56 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:50:57.373 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:56 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:50:57.373 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:56 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:50:58.236 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:57 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [7, 4]}]': finished 2024-04-02T04:50:58.236 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:57 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-04-02T04:50:58.236 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:57 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-04-02T04:50:58.236 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:57 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]': finished 2024-04-02T04:50:58.237 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:57 smithi192 ceph-mon[25697]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/368218600"}]': finished 2024-04-02T04:50:58.237 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:57 smithi192 ceph-mon[25697]: osdmap e78: 8 total, 8 up, 8 in 2024-04-02T04:50:58.237 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:57 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/2116005370' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/594554291"}]: dispatch 2024-04-02T04:50:58.260 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:57 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [7, 4]}]': finished 2024-04-02T04:50:58.260 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:57 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-04-02T04:50:58.260 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:57 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-04-02T04:50:58.260 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:57 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]': finished 2024-04-02T04:50:58.260 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:57 smithi086 ceph-mon[29505]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/368218600"}]': finished 2024-04-02T04:50:58.260 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:57 smithi086 ceph-mon[29505]: osdmap e78: 8 total, 8 up, 8 in 2024-04-02T04:50:58.260 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:57 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/2116005370' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/594554291"}]: dispatch 2024-04-02T04:50:58.260 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:57 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [7, 4]}]': finished 2024-04-02T04:50:58.260 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:57 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-04-02T04:50:58.261 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:57 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-04-02T04:50:58.261 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:57 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]': finished 2024-04-02T04:50:58.261 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:57 smithi086 ceph-mon[24811]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/368218600"}]': finished 2024-04-02T04:50:58.261 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:57 smithi086 ceph-mon[24811]: osdmap e78: 8 total, 8 up, 8 in 2024-04-02T04:50:58.261 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:57 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/2116005370' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/594554291"}]: dispatch 2024-04-02T04:50:59.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:58 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/2116005370' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/594554291"}]': finished 2024-04-02T04:50:59.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:58 smithi192 ceph-mon[25697]: osdmap e79: 8 total, 8 up, 8 in 2024-04-02T04:50:59.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:58 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/2241437352' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/939688624"}]: dispatch 2024-04-02T04:50:59.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:58 smithi192 ceph-mon[25697]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/939688624"}]: dispatch 2024-04-02T04:50:59.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:58 smithi192 ceph-mon[25697]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 74 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s rd, 383 B/s wr, 71 op/s 2024-04-02T04:50:59.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:58 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/2116005370' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/594554291"}]': finished 2024-04-02T04:50:59.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:58 smithi086 ceph-mon[24811]: osdmap e79: 8 total, 8 up, 8 in 2024-04-02T04:50:59.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:58 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/2241437352' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/939688624"}]: dispatch 2024-04-02T04:50:59.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:58 smithi086 ceph-mon[24811]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/939688624"}]: dispatch 2024-04-02T04:50:59.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:58 smithi086 ceph-mon[24811]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 74 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s rd, 383 B/s wr, 71 op/s 2024-04-02T04:50:59.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:58 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/2116005370' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/594554291"}]': finished 2024-04-02T04:50:59.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:58 smithi086 ceph-mon[29505]: osdmap e79: 8 total, 8 up, 8 in 2024-04-02T04:50:59.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:58 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/2241437352' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/939688624"}]: dispatch 2024-04-02T04:50:59.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:58 smithi086 ceph-mon[29505]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/939688624"}]: dispatch 2024-04-02T04:50:59.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:58 smithi086 ceph-mon[29505]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 74 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s rd, 383 B/s wr, 71 op/s 2024-04-02T04:51:00.333 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:51:00 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:51:00] "GET /metrics HTTP/1.1" 200 208022 "" "Prometheus/2.33.4" 2024-04-02T04:51:00.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[24811]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/939688624"}]': finished 2024-04-02T04:51:00.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[24811]: osdmap e80: 8 total, 8 up, 8 in 2024-04-02T04:51:00.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:51:00.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/4293134819' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1928070731"}]: dispatch 2024-04-02T04:51:00.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:51:00.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:51:00.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T04:51:00.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[24811]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T04:51:00.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[24811]: Adding iSCSI gateway http://:@172.21.15.86:5000 to Dashboard 2024-04-02T04:51:00.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T04:51:00.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[24811]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T04:51:00.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:51:00.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi086"}]: dispatch 2024-04-02T04:51:00.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[24811]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi086"}]: dispatch 2024-04-02T04:51:00.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:51:00.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:51:00.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:51:00.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[24811]: Checking dashboard <-> RGW credentials 2024-04-02T04:51:00.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:51:00.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[29505]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/939688624"}]': finished 2024-04-02T04:51:00.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[29505]: osdmap e80: 8 total, 8 up, 8 in 2024-04-02T04:51:00.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:51:00.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/4293134819' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1928070731"}]: dispatch 2024-04-02T04:51:00.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:51:00.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:51:00.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T04:51:00.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[29505]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T04:51:00.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[29505]: Adding iSCSI gateway http://:@172.21.15.86:5000 to Dashboard 2024-04-02T04:51:00.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T04:51:00.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[29505]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T04:51:00.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:51:00.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi086"}]: dispatch 2024-04-02T04:51:00.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[29505]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi086"}]: dispatch 2024-04-02T04:51:00.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:51:00.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:51:00.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:51:00.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[29505]: Checking dashboard <-> RGW credentials 2024-04-02T04:51:00.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:50:59 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:51:00.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:59 smithi192 ceph-mon[25697]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/939688624"}]': finished 2024-04-02T04:51:00.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:59 smithi192 ceph-mon[25697]: osdmap e80: 8 total, 8 up, 8 in 2024-04-02T04:51:00.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:59 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:51:00.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:59 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/4293134819' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1928070731"}]: dispatch 2024-04-02T04:51:00.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:59 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:51:00.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:59 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:51:00.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:59 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T04:51:00.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:59 smithi192 ceph-mon[25697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T04:51:00.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:59 smithi192 ceph-mon[25697]: Adding iSCSI gateway http://:@172.21.15.86:5000 to Dashboard 2024-04-02T04:51:00.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:59 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T04:51:00.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:59 smithi192 ceph-mon[25697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T04:51:00.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:59 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:51:00.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:59 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi086"}]: dispatch 2024-04-02T04:51:00.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:59 smithi192 ceph-mon[25697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi086"}]: dispatch 2024-04-02T04:51:00.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:59 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:51:00.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:59 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:51:00.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:59 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:51:00.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:59 smithi192 ceph-mon[25697]: Checking dashboard <-> RGW credentials 2024-04-02T04:51:00.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:50:59 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:51:01.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:01 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/4293134819' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1928070731"}]': finished 2024-04-02T04:51:01.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:01 smithi086 ceph-mon[24811]: osdmap e81: 8 total, 8 up, 8 in 2024-04-02T04:51:01.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:01 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/3252899301' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2885713909"}]: dispatch 2024-04-02T04:51:01.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:01 smithi086 ceph-mon[24811]: pgmap v87: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 74 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 767 B/s wr, 13 op/s 2024-04-02T04:51:01.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:01 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/4293134819' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1928070731"}]': finished 2024-04-02T04:51:01.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:01 smithi086 ceph-mon[29505]: osdmap e81: 8 total, 8 up, 8 in 2024-04-02T04:51:01.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:01 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/3252899301' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2885713909"}]: dispatch 2024-04-02T04:51:01.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:01 smithi086 ceph-mon[29505]: pgmap v87: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 74 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 767 B/s wr, 13 op/s 2024-04-02T04:51:01.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:01 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/4293134819' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1928070731"}]': finished 2024-04-02T04:51:01.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:01 smithi192 ceph-mon[25697]: osdmap e81: 8 total, 8 up, 8 in 2024-04-02T04:51:01.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:01 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/3252899301' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2885713909"}]: dispatch 2024-04-02T04:51:01.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:01 smithi192 ceph-mon[25697]: pgmap v87: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 74 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 767 B/s wr, 13 op/s 2024-04-02T04:51:02.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:02 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/3252899301' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2885713909"}]': finished 2024-04-02T04:51:02.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:02 smithi086 ceph-mon[29505]: osdmap e82: 8 total, 8 up, 8 in 2024-04-02T04:51:02.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:02 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/808738866' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/3311785205"}]: dispatch 2024-04-02T04:51:02.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:02 smithi086 ceph-mon[29505]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/3311785205"}]: dispatch 2024-04-02T04:51:02.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:02 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/3252899301' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2885713909"}]': finished 2024-04-02T04:51:02.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:02 smithi086 ceph-mon[24811]: osdmap e82: 8 total, 8 up, 8 in 2024-04-02T04:51:02.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:02 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/808738866' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/3311785205"}]: dispatch 2024-04-02T04:51:02.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:02 smithi086 ceph-mon[24811]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/3311785205"}]: dispatch 2024-04-02T04:51:02.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:02 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/3252899301' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2885713909"}]': finished 2024-04-02T04:51:02.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:02 smithi192 ceph-mon[25697]: osdmap e82: 8 total, 8 up, 8 in 2024-04-02T04:51:02.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:02 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/808738866' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/3311785205"}]: dispatch 2024-04-02T04:51:02.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:02 smithi192 ceph-mon[25697]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/3311785205"}]: dispatch 2024-04-02T04:51:03.524 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:03 smithi086 ceph-mon[29505]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/3311785205"}]': finished 2024-04-02T04:51:03.524 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:03 smithi086 ceph-mon[29505]: osdmap e83: 8 total, 8 up, 8 in 2024-04-02T04:51:03.524 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:03 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/2015101939' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2853453946"}]: dispatch 2024-04-02T04:51:03.524 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:03 smithi086 ceph-mon[29505]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2853453946"}]: dispatch 2024-04-02T04:51:03.524 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:03 smithi086 ceph-mon[29505]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s; 11 B/s, 0 objects/s recovering 2024-04-02T04:51:03.525 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:03 smithi086 ceph-mon[24811]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/3311785205"}]': finished 2024-04-02T04:51:03.525 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:03 smithi086 ceph-mon[24811]: osdmap e83: 8 total, 8 up, 8 in 2024-04-02T04:51:03.525 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:03 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/2015101939' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2853453946"}]: dispatch 2024-04-02T04:51:03.525 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:03 smithi086 ceph-mon[24811]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2853453946"}]: dispatch 2024-04-02T04:51:03.525 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:03 smithi086 ceph-mon[24811]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s; 11 B/s, 0 objects/s recovering 2024-04-02T04:51:03.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:03 smithi192 ceph-mon[25697]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/3311785205"}]': finished 2024-04-02T04:51:03.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:03 smithi192 ceph-mon[25697]: osdmap e83: 8 total, 8 up, 8 in 2024-04-02T04:51:03.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:03 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/2015101939' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2853453946"}]: dispatch 2024-04-02T04:51:03.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:03 smithi192 ceph-mon[25697]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2853453946"}]: dispatch 2024-04-02T04:51:03.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:03 smithi192 ceph-mon[25697]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s; 11 B/s, 0 objects/s recovering 2024-04-02T04:51:03.833 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:51:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:51:03.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.86:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.86 because it doesn't contain any IP SANs" 2024-04-02T04:51:03.834 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:51:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:51:03.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.192:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.192 because it doesn't contain any IP SANs" 2024-04-02T04:51:04.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:04 smithi086 ceph-mon[29505]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2853453946"}]': finished 2024-04-02T04:51:04.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:04 smithi086 ceph-mon[29505]: osdmap e84: 8 total, 8 up, 8 in 2024-04-02T04:51:04.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:04 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/864168990' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/3904718154"}]: dispatch 2024-04-02T04:51:04.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:04 smithi086 ceph-mon[24811]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2853453946"}]': finished 2024-04-02T04:51:04.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:04 smithi086 ceph-mon[24811]: osdmap e84: 8 total, 8 up, 8 in 2024-04-02T04:51:04.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:04 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/864168990' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/3904718154"}]: dispatch 2024-04-02T04:51:04.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:04 smithi192 ceph-mon[25697]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2853453946"}]': finished 2024-04-02T04:51:04.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:04 smithi192 ceph-mon[25697]: osdmap e84: 8 total, 8 up, 8 in 2024-04-02T04:51:04.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:04 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/864168990' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/3904718154"}]: dispatch 2024-04-02T04:51:05.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:05 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/864168990' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/3904718154"}]': finished 2024-04-02T04:51:05.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:05 smithi086 ceph-mon[24811]: osdmap e85: 8 total, 8 up, 8 in 2024-04-02T04:51:05.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:05 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1447038358' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/3904718154"}]: dispatch 2024-04-02T04:51:05.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:05 smithi086 ceph-mon[24811]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s; 11 B/s, 0 objects/s recovering 2024-04-02T04:51:05.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:05 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/864168990' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/3904718154"}]': finished 2024-04-02T04:51:05.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:05 smithi086 ceph-mon[29505]: osdmap e85: 8 total, 8 up, 8 in 2024-04-02T04:51:05.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:05 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1447038358' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/3904718154"}]: dispatch 2024-04-02T04:51:05.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:05 smithi086 ceph-mon[29505]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s; 11 B/s, 0 objects/s recovering 2024-04-02T04:51:05.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:51:05 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:51:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:51:05.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:05 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/864168990' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/3904718154"}]': finished 2024-04-02T04:51:05.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:05 smithi192 ceph-mon[25697]: osdmap e85: 8 total, 8 up, 8 in 2024-04-02T04:51:05.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:05 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1447038358' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/3904718154"}]: dispatch 2024-04-02T04:51:05.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:05 smithi192 ceph-mon[25697]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s; 11 B/s, 0 objects/s recovering 2024-04-02T04:51:06.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:06 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1447038358' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/3904718154"}]': finished 2024-04-02T04:51:06.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:06 smithi086 ceph-mon[29505]: osdmap e86: 8 total, 8 up, 8 in 2024-04-02T04:51:06.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:06 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/137169587' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/4053196327"}]: dispatch 2024-04-02T04:51:06.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:06 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1447038358' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/3904718154"}]': finished 2024-04-02T04:51:06.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:06 smithi086 ceph-mon[24811]: osdmap e86: 8 total, 8 up, 8 in 2024-04-02T04:51:06.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:06 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/137169587' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/4053196327"}]: dispatch 2024-04-02T04:51:06.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:06 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1447038358' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/3904718154"}]': finished 2024-04-02T04:51:06.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:06 smithi192 ceph-mon[25697]: osdmap e86: 8 total, 8 up, 8 in 2024-04-02T04:51:06.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:06 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/137169587' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/4053196327"}]: dispatch 2024-04-02T04:51:07.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:07 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:51:07.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:07 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/137169587' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/4053196327"}]': finished 2024-04-02T04:51:07.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:07 smithi086 ceph-mon[29505]: osdmap e87: 8 total, 8 up, 8 in 2024-04-02T04:51:07.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:07 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1373966895' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1081336665"}]: dispatch 2024-04-02T04:51:07.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:07 smithi086 ceph-mon[29505]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:51:07.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:07 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:51:07.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:07 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/137169587' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/4053196327"}]': finished 2024-04-02T04:51:07.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:07 smithi086 ceph-mon[24811]: osdmap e87: 8 total, 8 up, 8 in 2024-04-02T04:51:07.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:07 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1373966895' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1081336665"}]: dispatch 2024-04-02T04:51:07.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:07 smithi086 ceph-mon[24811]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:51:07.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:07 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:51:07.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:07 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/137169587' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/4053196327"}]': finished 2024-04-02T04:51:07.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:07 smithi192 ceph-mon[25697]: osdmap e87: 8 total, 8 up, 8 in 2024-04-02T04:51:07.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:07 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1373966895' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1081336665"}]: dispatch 2024-04-02T04:51:07.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:07 smithi192 ceph-mon[25697]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:51:08.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:08 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1373966895' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1081336665"}]': finished 2024-04-02T04:51:08.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:08 smithi086 ceph-mon[29505]: osdmap e88: 8 total, 8 up, 8 in 2024-04-02T04:51:08.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:08 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/159826101' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/1417258151"}]: dispatch 2024-04-02T04:51:08.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:08 smithi086 ceph-mon[29505]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/1417258151"}]: dispatch 2024-04-02T04:51:08.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:08 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1373966895' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1081336665"}]': finished 2024-04-02T04:51:08.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:08 smithi086 ceph-mon[24811]: osdmap e88: 8 total, 8 up, 8 in 2024-04-02T04:51:08.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:08 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/159826101' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/1417258151"}]: dispatch 2024-04-02T04:51:08.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:08 smithi086 ceph-mon[24811]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/1417258151"}]: dispatch 2024-04-02T04:51:08.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:08 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1373966895' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1081336665"}]': finished 2024-04-02T04:51:08.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:08 smithi192 ceph-mon[25697]: osdmap e88: 8 total, 8 up, 8 in 2024-04-02T04:51:08.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:08 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/159826101' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/1417258151"}]: dispatch 2024-04-02T04:51:08.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:08 smithi192 ceph-mon[25697]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/1417258151"}]: dispatch 2024-04-02T04:51:09.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:09 smithi086 ceph-mon[29505]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/1417258151"}]': finished 2024-04-02T04:51:09.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:09 smithi086 ceph-mon[29505]: osdmap e89: 8 total, 8 up, 8 in 2024-04-02T04:51:09.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:09 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1757649924' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/939688624"}]: dispatch 2024-04-02T04:51:09.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:09 smithi086 ceph-mon[29505]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/939688624"}]: dispatch 2024-04-02T04:51:09.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:09 smithi086 ceph-mon[29505]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:51:09.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:09 smithi086 ceph-mon[24811]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/1417258151"}]': finished 2024-04-02T04:51:09.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:09 smithi086 ceph-mon[24811]: osdmap e89: 8 total, 8 up, 8 in 2024-04-02T04:51:09.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:09 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1757649924' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/939688624"}]: dispatch 2024-04-02T04:51:09.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:09 smithi086 ceph-mon[24811]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/939688624"}]: dispatch 2024-04-02T04:51:09.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:09 smithi086 ceph-mon[24811]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:51:09.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:09 smithi192 ceph-mon[25697]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/1417258151"}]': finished 2024-04-02T04:51:09.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:09 smithi192 ceph-mon[25697]: osdmap e89: 8 total, 8 up, 8 in 2024-04-02T04:51:09.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:09 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1757649924' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/939688624"}]: dispatch 2024-04-02T04:51:09.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:09 smithi192 ceph-mon[25697]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/939688624"}]: dispatch 2024-04-02T04:51:09.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:09 smithi192 ceph-mon[25697]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:51:10.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:51:10 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:51:10] "GET /metrics HTTP/1.1" 200 208022 "" "Prometheus/2.33.4" 2024-04-02T04:51:10.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:10 smithi086 ceph-mon[24811]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/939688624"}]': finished 2024-04-02T04:51:10.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:10 smithi086 ceph-mon[24811]: osdmap e90: 8 total, 8 up, 8 in 2024-04-02T04:51:10.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:10 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/3458491553' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2634385848"}]: dispatch 2024-04-02T04:51:10.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:10 smithi086 ceph-mon[24811]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2634385848"}]: dispatch 2024-04-02T04:51:10.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:10 smithi086 ceph-mon[29505]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/939688624"}]': finished 2024-04-02T04:51:10.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:10 smithi086 ceph-mon[29505]: osdmap e90: 8 total, 8 up, 8 in 2024-04-02T04:51:10.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:10 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/3458491553' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2634385848"}]: dispatch 2024-04-02T04:51:10.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:10 smithi086 ceph-mon[29505]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2634385848"}]: dispatch 2024-04-02T04:51:10.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:10 smithi192 ceph-mon[25697]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/939688624"}]': finished 2024-04-02T04:51:10.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:10 smithi192 ceph-mon[25697]: osdmap e90: 8 total, 8 up, 8 in 2024-04-02T04:51:10.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:10 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/3458491553' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2634385848"}]: dispatch 2024-04-02T04:51:10.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:10 smithi192 ceph-mon[25697]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2634385848"}]: dispatch 2024-04-02T04:51:11.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:11 smithi086 ceph-mon[29505]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2634385848"}]': finished 2024-04-02T04:51:11.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:11 smithi086 ceph-mon[29505]: osdmap e91: 8 total, 8 up, 8 in 2024-04-02T04:51:11.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:11 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1447157422' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1565660562"}]: dispatch 2024-04-02T04:51:11.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:11 smithi086 ceph-mon[29505]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1565660562"}]: dispatch 2024-04-02T04:51:11.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:11 smithi086 ceph-mon[29505]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:51:11.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:11 smithi086 ceph-mon[24811]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2634385848"}]': finished 2024-04-02T04:51:11.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:11 smithi086 ceph-mon[24811]: osdmap e91: 8 total, 8 up, 8 in 2024-04-02T04:51:11.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:11 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1447157422' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1565660562"}]: dispatch 2024-04-02T04:51:11.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:11 smithi086 ceph-mon[24811]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1565660562"}]: dispatch 2024-04-02T04:51:11.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:11 smithi086 ceph-mon[24811]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:51:11.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:11 smithi192 ceph-mon[25697]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2634385848"}]': finished 2024-04-02T04:51:11.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:11 smithi192 ceph-mon[25697]: osdmap e91: 8 total, 8 up, 8 in 2024-04-02T04:51:11.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:11 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1447157422' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1565660562"}]: dispatch 2024-04-02T04:51:11.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:11 smithi192 ceph-mon[25697]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1565660562"}]: dispatch 2024-04-02T04:51:11.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:11 smithi192 ceph-mon[25697]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:51:12.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:12 smithi086 ceph-mon[29505]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1565660562"}]': finished 2024-04-02T04:51:12.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:12 smithi086 ceph-mon[29505]: osdmap e92: 8 total, 8 up, 8 in 2024-04-02T04:51:12.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:12 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/2864635212' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/1417258151"}]: dispatch 2024-04-02T04:51:12.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:12 smithi086 ceph-mon[24811]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1565660562"}]': finished 2024-04-02T04:51:12.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:12 smithi086 ceph-mon[24811]: osdmap e92: 8 total, 8 up, 8 in 2024-04-02T04:51:12.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:12 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/2864635212' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/1417258151"}]: dispatch 2024-04-02T04:51:12.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:12 smithi192 ceph-mon[25697]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1565660562"}]': finished 2024-04-02T04:51:12.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:12 smithi192 ceph-mon[25697]: osdmap e92: 8 total, 8 up, 8 in 2024-04-02T04:51:12.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:12 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/2864635212' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/1417258151"}]: dispatch 2024-04-02T04:51:13.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:13 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/2864635212' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/1417258151"}]': finished 2024-04-02T04:51:13.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:13 smithi086 ceph-mon[24811]: osdmap e93: 8 total, 8 up, 8 in 2024-04-02T04:51:13.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:13 smithi086 ceph-mon[24811]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:51:13.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:13 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/2864635212' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/1417258151"}]': finished 2024-04-02T04:51:13.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:13 smithi086 ceph-mon[29505]: osdmap e93: 8 total, 8 up, 8 in 2024-04-02T04:51:13.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:13 smithi086 ceph-mon[29505]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:51:13.584 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:51:13 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=error ts=2024-04-02T04:51:13.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.86:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.86 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.192:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.192 because it doesn't contain any IP SANs" 2024-04-02T04:51:13.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:13 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/2864635212' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/1417258151"}]': finished 2024-04-02T04:51:13.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:13 smithi192 ceph-mon[25697]: osdmap e93: 8 total, 8 up, 8 in 2024-04-02T04:51:13.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:13 smithi192 ceph-mon[25697]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:51:15.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:51:15 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:51:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:51:16.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:15 smithi086 ceph-mon[24811]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 941 B/s rd, 0 op/s 2024-04-02T04:51:16.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:15 smithi086 ceph-mon[29505]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 941 B/s rd, 0 op/s 2024-04-02T04:51:16.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:15 smithi192 ceph-mon[25697]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 941 B/s rd, 0 op/s 2024-04-02T04:51:18.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:17 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:51:18.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:17 smithi086 ceph-mon[24811]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 795 B/s rd, 0 op/s 2024-04-02T04:51:18.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:17 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:51:18.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:17 smithi086 ceph-mon[29505]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 795 B/s rd, 0 op/s 2024-04-02T04:51:18.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:17 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:51:18.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:17 smithi192 ceph-mon[25697]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 795 B/s rd, 0 op/s 2024-04-02T04:51:20.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:19 smithi086 ceph-mon[29505]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:51:20.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:19 smithi086 ceph-mon[24811]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:51:20.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:19 smithi192 ceph-mon[25697]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:51:20.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:51:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:51:20] "GET /metrics HTTP/1.1" 200 208133 "" "Prometheus/2.33.4" 2024-04-02T04:51:21.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:21 smithi192 ceph-mon[25697]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T04:51:22.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:21 smithi086 ceph-mon[24811]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T04:51:22.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:21 smithi086 ceph-mon[29505]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T04:51:24.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:23 smithi086 ceph-mon[24811]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-02T04:51:24.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:23 smithi086 ceph-mon[29505]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-02T04:51:24.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:23 smithi192 ceph-mon[25697]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-02T04:51:25.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:51:25 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:51:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:51:26.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:25 smithi086 ceph-mon[24811]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:51:26.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:25 smithi086 ceph-mon[29505]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:51:26.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:25 smithi192 ceph-mon[25697]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:51:28.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:27 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:51:28.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:27 smithi086 ceph-mon[24811]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:51:28.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:27 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:51:28.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:27 smithi086 ceph-mon[29505]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:51:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:27 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:51:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:27 smithi192 ceph-mon[25697]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:51:30.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:29 smithi086 ceph-mon[24811]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:51:30.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:29 smithi086 ceph-mon[29505]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:51:30.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:29 smithi192 ceph-mon[25697]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:51:30.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:51:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:51:30] "GET /metrics HTTP/1.1" 200 208090 "" "Prometheus/2.33.4" 2024-04-02T04:51:31.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:31 smithi192 ceph-mon[25697]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:51:32.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:31 smithi086 ceph-mon[24811]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:51:32.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:31 smithi086 ceph-mon[29505]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:51:34.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:33 smithi086 ceph-mon[24811]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:51:34.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:33 smithi086 ceph-mon[29505]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:51:34.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:33 smithi192 ceph-mon[25697]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:51:35.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:51:35 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:51:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:51:36.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:35 smithi086 ceph-mon[24811]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:51:36.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:35 smithi086 ceph-mon[29505]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:51:36.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:35 smithi192 ceph-mon[25697]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:51:38.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:37 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:51:38.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:37 smithi086 ceph-mon[29505]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:51:38.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:37 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:51:38.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:37 smithi086 ceph-mon[24811]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:51:38.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:37 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:51:38.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:37 smithi192 ceph-mon[25697]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:51:40.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:39 smithi086 ceph-mon[24811]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:51:40.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:39 smithi086 ceph-mon[29505]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:51:40.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:39 smithi192 ceph-mon[25697]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:51:40.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:51:40 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:51:40] "GET /metrics HTTP/1.1" 200 208090 "" "Prometheus/2.33.4" 2024-04-02T04:51:41.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:41 smithi192 ceph-mon[25697]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:51:42.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:41 smithi086 ceph-mon[24811]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:51:42.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:41 smithi086 ceph-mon[29505]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:51:44.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:43 smithi086 ceph-mon[24811]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:51:44.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:43 smithi086 ceph-mon[29505]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:51:44.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:43 smithi192 ceph-mon[25697]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:51:45.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:51:45 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:51:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:51:46.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:45 smithi086 ceph-mon[24811]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:51:46.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:45 smithi086 ceph-mon[29505]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:51:46.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:45 smithi192 ceph-mon[25697]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:51:48.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:47 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:51:48.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:47 smithi086 ceph-mon[24811]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:51:48.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:47 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:51:48.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:47 smithi086 ceph-mon[29505]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:51:48.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:47 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:51:48.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:47 smithi192 ceph-mon[25697]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:51:50.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:49 smithi086 ceph-mon[29505]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:51:50.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:49 smithi086 ceph-mon[24811]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:51:50.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:49 smithi192 ceph-mon[25697]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:51:50.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:51:50 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:51:50] "GET /metrics HTTP/1.1" 200 208056 "" "Prometheus/2.33.4" 2024-04-02T04:51:51.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:51 smithi192 ceph-mon[25697]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:51:52.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:51 smithi086 ceph-mon[24811]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:51:52.088 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:51 smithi086 ceph-mon[29505]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:51:54.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:53 smithi086 ceph-mon[24811]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:51:54.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:53 smithi086 ceph-mon[29505]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:51:54.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:53 smithi192 ceph-mon[25697]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:51:55.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:51:55 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:51:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:51:56.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:55 smithi086 ceph-mon[24811]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:51:56.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:55 smithi086 ceph-mon[29505]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:51:56.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:55 smithi192 ceph-mon[25697]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:51:57.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:56 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:51:57.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:56 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:51:57.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:56 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:51:57.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:56 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:51:57.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:56 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:51:57.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:56 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:51:58.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:57 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:51:58.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:57 smithi086 ceph-mon[24811]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:51:58.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:57 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:51:58.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:57 smithi086 ceph-mon[29505]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:51:58.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:57 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:51:58.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:57 smithi192 ceph-mon[25697]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:00.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:51:59 smithi086 ceph-mon[29505]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:52:00.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:51:59 smithi086 ceph-mon[24811]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:52:00.154 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:51:59 smithi192 ceph-mon[25697]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:52:00.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:52:00 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:52:00] "GET /metrics HTTP/1.1" 200 208048 "" "Prometheus/2.33.4" 2024-04-02T04:52:01.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:00 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:52:01.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:00 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:52:01.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:00 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:52:01.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:00 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:52:01.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:00 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:52:01.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:00 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:52:01.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:00 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:52:01.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:00 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:52:01.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:00 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:52:02.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:01 smithi086 ceph-mon[24811]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:02.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:01 smithi086 ceph-mon[29505]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:02.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:01 smithi192 ceph-mon[25697]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:04.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:03 smithi086 ceph-mon[24811]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:52:04.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:03 smithi086 ceph-mon[29505]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:52:04.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:03 smithi192 ceph-mon[25697]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:52:05.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:52:05 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:52:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:52:06.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:05 smithi086 ceph-mon[24811]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:06.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:05 smithi086 ceph-mon[29505]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:05 smithi192 ceph-mon[25697]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:08.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:07 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:52:08.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:07 smithi086 ceph-mon[24811]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:08.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:07 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:52:08.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:07 smithi086 ceph-mon[29505]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:08.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:07 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:52:08.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:07 smithi192 ceph-mon[25697]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:10.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:09 smithi086 ceph-mon[24811]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:52:10.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:09 smithi086 ceph-mon[29505]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:52:10.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:09 smithi192 ceph-mon[25697]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:52:10.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:52:10 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:52:10] "GET /metrics HTTP/1.1" 200 208048 "" "Prometheus/2.33.4" 2024-04-02T04:52:12.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:11 smithi086 ceph-mon[24811]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:12.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:11 smithi086 ceph-mon[29505]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:12.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:11 smithi192 ceph-mon[25697]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:14.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:13 smithi086 ceph-mon[24811]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:52:14.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:13 smithi086 ceph-mon[29505]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:52:14.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:13 smithi192 ceph-mon[25697]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:52:15.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:52:15 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:52:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:52:16.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:15 smithi086 ceph-mon[24811]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:16.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:15 smithi086 ceph-mon[29505]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:16.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:15 smithi192 ceph-mon[25697]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:18.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:17 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:52:18.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:17 smithi086 ceph-mon[29505]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:18.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:17 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:52:18.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:17 smithi086 ceph-mon[24811]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:18.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:17 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:52:18.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:17 smithi192 ceph-mon[25697]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:20.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:19 smithi086 ceph-mon[24811]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:52:20.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:19 smithi086 ceph-mon[29505]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:52:20.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:19 smithi192 ceph-mon[25697]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:52:20.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:52:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:52:20] "GET /metrics HTTP/1.1" 200 208048 "" "Prometheus/2.33.4" 2024-04-02T04:52:22.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:21 smithi086 ceph-mon[29505]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:22.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:21 smithi086 ceph-mon[24811]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:22.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:21 smithi192 ceph-mon[25697]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:24.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:23 smithi086 ceph-mon[24811]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:52:24.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:23 smithi086 ceph-mon[29505]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:52:24.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:23 smithi192 ceph-mon[25697]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:52:25.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:52:25 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:52:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:52:26.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:25 smithi086 ceph-mon[24811]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:26.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:25 smithi086 ceph-mon[29505]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:26.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:25 smithi192 ceph-mon[25697]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:28.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:27 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:52:28.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:27 smithi086 ceph-mon[24811]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:28.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:27 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:52:28.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:27 smithi086 ceph-mon[29505]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:27 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:52:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:27 smithi192 ceph-mon[25697]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:30.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:29 smithi086 ceph-mon[24811]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:52:30.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:29 smithi086 ceph-mon[29505]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:52:30.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:29 smithi192 ceph-mon[25697]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:52:30.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:52:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:52:30] "GET /metrics HTTP/1.1" 200 208045 "" "Prometheus/2.33.4" 2024-04-02T04:52:32.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:31 smithi086 ceph-mon[24811]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:32.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:31 smithi086 ceph-mon[29505]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:32.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:31 smithi192 ceph-mon[25697]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:34.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:33 smithi086 ceph-mon[24811]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:52:34.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:33 smithi086 ceph-mon[29505]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:52:34.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:33 smithi192 ceph-mon[25697]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:52:35.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:52:35 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:52:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:52:36.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:35 smithi086 ceph-mon[24811]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:36.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:35 smithi086 ceph-mon[29505]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:36.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:35 smithi192 ceph-mon[25697]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:38.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:37 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:52:38.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:37 smithi086 ceph-mon[29505]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:38.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:37 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:52:38.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:37 smithi086 ceph-mon[24811]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:38.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:37 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:52:38.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:37 smithi192 ceph-mon[25697]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:40.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:39 smithi086 ceph-mon[29505]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:52:40.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:39 smithi086 ceph-mon[24811]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:52:40.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:39 smithi192 ceph-mon[25697]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:52:40.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:52:40 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:52:40] "GET /metrics HTTP/1.1" 200 208045 "" "Prometheus/2.33.4" 2024-04-02T04:52:42.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:41 smithi086 ceph-mon[29505]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:42.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:41 smithi086 ceph-mon[24811]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:42.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:41 smithi192 ceph-mon[25697]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:44.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:43 smithi086 ceph-mon[29505]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:52:44.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:43 smithi086 ceph-mon[24811]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:52:44.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:43 smithi192 ceph-mon[25697]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:52:45.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:52:45 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:52:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:52:46.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:45 smithi086 ceph-mon[29505]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:46.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:45 smithi086 ceph-mon[24811]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:46.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:45 smithi192 ceph-mon[25697]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:48.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:47 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:52:48.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:47 smithi086 ceph-mon[29505]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:48.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:47 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:52:48.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:47 smithi086 ceph-mon[24811]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:48.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:47 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:52:48.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:47 smithi192 ceph-mon[25697]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:50.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:49 smithi086 ceph-mon[29505]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:52:50.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:49 smithi086 ceph-mon[24811]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:52:50.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:49 smithi192 ceph-mon[25697]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:52:50.584 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:52:50 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:52:50] "GET /metrics HTTP/1.1" 200 208051 "" "Prometheus/2.33.4" 2024-04-02T04:52:52.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:51 smithi086 ceph-mon[24811]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:52.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:51 smithi086 ceph-mon[29505]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:52.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:51 smithi192 ceph-mon[25697]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:54.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:53 smithi086 ceph-mon[24811]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:52:54.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:53 smithi086 ceph-mon[29505]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:52:54.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:53 smithi192 ceph-mon[25697]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:52:55.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:52:55 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:52:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:52:56.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:55 smithi086 ceph-mon[29505]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:56.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:55 smithi086 ceph-mon[24811]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:56.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:55 smithi192 ceph-mon[25697]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:57.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:56 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:52:57.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:56 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:52:57.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:56 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:52:57.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:56 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:52:57.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:56 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:52:57.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:56 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:52:58.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:57 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:52:58.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:57 smithi086 ceph-mon[24811]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:58.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:57 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:52:58.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:57 smithi086 ceph-mon[29505]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:52:58.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:57 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:52:58.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:57 smithi192 ceph-mon[25697]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:00.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:52:59 smithi086 ceph-mon[24811]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:00.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:52:59 smithi086 ceph-mon[29505]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:00.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:52:59 smithi192 ceph-mon[25697]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:00.490 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:53:00 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:53:00] "GET /metrics HTTP/1.1" 200 208045 "" "Prometheus/2.33.4" 2024-04-02T04:53:01.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:00 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:53:01.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:00 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:53:01.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:00 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:53:01.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:00 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:53:01.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:00 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:53:01.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:00 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:53:01.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:00 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:53:01.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:00 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:53:01.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:00 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:53:02.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:01 smithi086 ceph-mon[29505]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:02.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:01 smithi086 ceph-mon[24811]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:02.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:01 smithi192 ceph-mon[25697]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:04.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:03 smithi086 ceph-mon[24811]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:04.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:03 smithi086 ceph-mon[29505]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:04.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:03 smithi192 ceph-mon[25697]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:05.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:53:05 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:53:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:53:06.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:05 smithi086 ceph-mon[24811]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:06.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:05 smithi086 ceph-mon[29505]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:05 smithi192 ceph-mon[25697]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:08.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:07 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:53:08.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:07 smithi086 ceph-mon[29505]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:08.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:07 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:53:08.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:07 smithi086 ceph-mon[24811]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:08.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:07 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:53:08.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:07 smithi192 ceph-mon[25697]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:10.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:09 smithi086 ceph-mon[29505]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:10.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:09 smithi086 ceph-mon[24811]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:10.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:09 smithi192 ceph-mon[25697]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:10.584 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:53:10 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:53:10] "GET /metrics HTTP/1.1" 200 208045 "" "Prometheus/2.33.4" 2024-04-02T04:53:12.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:11 smithi086 ceph-mon[29505]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:12.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:11 smithi086 ceph-mon[24811]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:12.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:11 smithi192 ceph-mon[25697]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:14.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:13 smithi086 ceph-mon[24811]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:14.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:13 smithi086 ceph-mon[29505]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:14.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:13 smithi192 ceph-mon[25697]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:15.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:53:15 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:53:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:53:16.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:15 smithi086 ceph-mon[29505]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:16.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:15 smithi086 ceph-mon[24811]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:16.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:15 smithi192 ceph-mon[25697]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:18.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:17 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:53:18.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:17 smithi086 ceph-mon[29505]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:18.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:17 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:53:18.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:17 smithi086 ceph-mon[24811]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:18.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:17 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:53:18.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:17 smithi192 ceph-mon[25697]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:20.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:19 smithi086 ceph-mon[24811]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:20.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:19 smithi086 ceph-mon[29505]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:20.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:19 smithi192 ceph-mon[25697]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:20.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:53:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:53:20] "GET /metrics HTTP/1.1" 200 208037 "" "Prometheus/2.33.4" 2024-04-02T04:53:22.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:21 smithi086 ceph-mon[24811]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:22.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:21 smithi086 ceph-mon[29505]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:22.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:21 smithi192 ceph-mon[25697]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:24.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:23 smithi086 ceph-mon[24811]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:24.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:23 smithi086 ceph-mon[29505]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:24.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:23 smithi192 ceph-mon[25697]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:25.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:53:25 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:53:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:53:26.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:25 smithi086 ceph-mon[24811]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:26.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:25 smithi086 ceph-mon[29505]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:26.091 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:25 smithi192 ceph-mon[25697]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:28.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:27 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:53:28.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:27 smithi086 ceph-mon[29505]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:28.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:27 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:53:28.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:27 smithi086 ceph-mon[24811]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:28.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:27 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:53:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:27 smithi192 ceph-mon[25697]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:30.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:29 smithi086 ceph-mon[29505]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:30.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:29 smithi086 ceph-mon[24811]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:30.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:29 smithi192 ceph-mon[25697]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:30.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:53:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:53:30] "GET /metrics HTTP/1.1" 200 207999 "" "Prometheus/2.33.4" 2024-04-02T04:53:32.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:31 smithi086 ceph-mon[24811]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:32.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:31 smithi086 ceph-mon[29505]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:32.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:31 smithi192 ceph-mon[25697]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:34.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:33 smithi086 ceph-mon[24811]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:34.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:33 smithi086 ceph-mon[29505]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:34.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:33 smithi192 ceph-mon[25697]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:35.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:53:35 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:53:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:53:36.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:35 smithi086 ceph-mon[29505]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:36.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:35 smithi086 ceph-mon[24811]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:36.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:35 smithi192 ceph-mon[25697]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:38.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:37 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:53:38.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:37 smithi086 ceph-mon[29505]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:38.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:37 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:53:38.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:37 smithi086 ceph-mon[24811]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:38.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:37 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:53:38.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:37 smithi192 ceph-mon[25697]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:40.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:39 smithi086 ceph-mon[24811]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:40.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:39 smithi086 ceph-mon[29505]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:40.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:39 smithi192 ceph-mon[25697]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:40.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:53:40 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:53:40] "GET /metrics HTTP/1.1" 200 207999 "" "Prometheus/2.33.4" 2024-04-02T04:53:42.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:41 smithi086 ceph-mon[24811]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:42.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:41 smithi086 ceph-mon[29505]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:42.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:41 smithi192 ceph-mon[25697]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:44.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:43 smithi086 ceph-mon[24811]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:44.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:43 smithi086 ceph-mon[29505]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:44.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:43 smithi192 ceph-mon[25697]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:45.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:53:45 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:53:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:53:46.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:45 smithi086 ceph-mon[24811]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:46.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:45 smithi086 ceph-mon[29505]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:46.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:45 smithi192 ceph-mon[25697]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:48.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:47 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:53:48.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:47 smithi086 ceph-mon[29505]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:48.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:47 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:53:48.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:47 smithi086 ceph-mon[24811]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:48.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:47 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:53:48.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:47 smithi192 ceph-mon[25697]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:50.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:49 smithi086 ceph-mon[29505]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:50.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:49 smithi086 ceph-mon[24811]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:50.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:49 smithi192 ceph-mon[25697]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:50.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:53:50 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:53:50] "GET /metrics HTTP/1.1" 200 207997 "" "Prometheus/2.33.4" 2024-04-02T04:53:52.070 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:51 smithi086 ceph-mon[24811]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:52.070 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:51 smithi086 ceph-mon[29505]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:52.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:51 smithi192 ceph-mon[25697]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:52.732 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-04-02T04:53:53.926 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:53 smithi086 ceph-mon[24811]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:53.926 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:53 smithi086 ceph-mon[29505]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:54.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:53 smithi192 ceph-mon[25697]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:55.171 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-04-02T04:53:55.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:53:55 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:53:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:53:56.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:55 smithi086 ceph-mon[24811]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:56.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:55 smithi086 ceph-mon[29505]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:56.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:55 smithi192 ceph-mon[25697]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:57.045 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:56 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:53:57.045 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:56 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:53:57.045 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:56 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:53:57.046 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:56 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:53:57.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:56 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:53:57.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:56 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:53:57.659 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph config set global log_to_journald false --force' 2024-04-02T04:53:58.069 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:57 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:53:58.069 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:57 smithi086 ceph-mon[29505]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:58.069 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:57 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:53:58.069 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:57 smithi086 ceph-mon[24811]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:58.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:57 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:53:58.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:57 smithi192 ceph-mon[25697]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:53:59.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:53:59 smithi086 ceph-mon[24811]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:53:59.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:53:59 smithi086 ceph-mon[29505]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:54:00.112 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch ps' 2024-04-02T04:54:00.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:53:59 smithi192 ceph-mon[25697]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:54:00.415 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:54:00 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:54:00] "GET /metrics HTTP/1.1" 200 207996 "" "Prometheus/2.33.4" 2024-04-02T04:54:01.008 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:00 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:54:01.008 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:00 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:54:01.008 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:00 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:54:01.008 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:00 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:54:01.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:00 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:54:01.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:00 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:54:02.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:01 smithi086 ceph-mon[29505]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:02.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:01 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:54:02.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:01 smithi086 ceph-mon[24811]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:02.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:01 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:54:02.144 INFO:teuthology.orchestra.run.smithi086.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T04:54:02.144 INFO:teuthology.orchestra.run.smithi086.stdout:alertmanager.a smithi086 *:9093,9094 running (3m) 3m ago 4m 14.0M - ba2b418f427c 24a36455c114 2024-04-02T04:54:02.144 INFO:teuthology.orchestra.run.smithi086.stdout:grafana.a smithi192 *:3000 running (3m) 3m ago 3m 44.8M - 8.3.5 dad864ee21e9 6517aa8b3121 2024-04-02T04:54:02.144 INFO:teuthology.orchestra.run.smithi086.stdout:iscsi.foo.smithi086.rsbqwx smithi086 running (3m) 3m ago 3m 66.8M - 3.5 e1d6a67b021e 828835e36fd4 2024-04-02T04:54:02.144 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.x smithi192 *:8443 running (8m) 3m ago 8m 410M - 17.2.0 e1d6a67b021e 2ebae22b0016 2024-04-02T04:54:02.144 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.y smithi086 *:9283 running (10m) 3m ago 10m 453M - 17.2.0 e1d6a67b021e 02f336a19fd6 2024-04-02T04:54:02.144 INFO:teuthology.orchestra.run.smithi086.stdout:mon.a smithi086 running (10m) 3m ago 10m 46.6M 2048M 17.2.0 e1d6a67b021e b6c507ab96f8 2024-04-02T04:54:02.144 INFO:teuthology.orchestra.run.smithi086.stdout:mon.b smithi192 running (8m) 3m ago 8m 40.9M 2048M 17.2.0 e1d6a67b021e 4dadd8d1c318 2024-04-02T04:54:02.145 INFO:teuthology.orchestra.run.smithi086.stdout:mon.c smithi086 running (8m) 3m ago 8m 38.0M 2048M 17.2.0 e1d6a67b021e 49275483a2d4 2024-04-02T04:54:02.145 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.a smithi086 *:9100 running (4m) 3m ago 4m 11.1M - 1dbe0e931976 0151f4e72bd9 2024-04-02T04:54:02.145 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.b smithi192 *:9100 running (4m) 3m ago 4m 15.1M - 1dbe0e931976 27476f65f84f 2024-04-02T04:54:02.145 INFO:teuthology.orchestra.run.smithi086.stdout:osd.0 smithi086 running (8m) 3m ago 8m 51.6M 4016M 17.2.0 e1d6a67b021e 6d90c60d1c71 2024-04-02T04:54:02.145 INFO:teuthology.orchestra.run.smithi086.stdout:osd.1 smithi086 running (7m) 3m ago 7m 49.7M 4016M 17.2.0 e1d6a67b021e 203689f71603 2024-04-02T04:54:02.145 INFO:teuthology.orchestra.run.smithi086.stdout:osd.2 smithi086 running (7m) 3m ago 7m 43.8M 4016M 17.2.0 e1d6a67b021e 472c860a9af8 2024-04-02T04:54:02.145 INFO:teuthology.orchestra.run.smithi086.stdout:osd.3 smithi086 running (6m) 3m ago 6m 45.2M 4016M 17.2.0 e1d6a67b021e 327538dac44e 2024-04-02T04:54:02.145 INFO:teuthology.orchestra.run.smithi086.stdout:osd.4 smithi192 running (6m) 3m ago 6m 45.9M 4272M 17.2.0 e1d6a67b021e 6535d7919a2d 2024-04-02T04:54:02.145 INFO:teuthology.orchestra.run.smithi086.stdout:osd.5 smithi192 running (6m) 3m ago 6m 46.6M 4272M 17.2.0 e1d6a67b021e 7259249605f2 2024-04-02T04:54:02.145 INFO:teuthology.orchestra.run.smithi086.stdout:osd.6 smithi192 running (5m) 3m ago 5m 44.2M 4272M 17.2.0 e1d6a67b021e bb7d171b02a1 2024-04-02T04:54:02.145 INFO:teuthology.orchestra.run.smithi086.stdout:osd.7 smithi192 running (5m) 3m ago 5m 46.1M 4272M 17.2.0 e1d6a67b021e 95f5a0353999 2024-04-02T04:54:02.145 INFO:teuthology.orchestra.run.smithi086.stdout:prometheus.a smithi192 *:9095 running (3m) 3m ago 4m 44.5M - 514e6a882f6e 782ae1d5e7f9 2024-04-02T04:54:02.145 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi086.bxzxlp smithi086 *:8000 running (3m) 3m ago 3m 71.5M - 17.2.0 e1d6a67b021e 6725785fbc53 2024-04-02T04:54:02.145 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi192.pmlbot smithi192 *:8000 running (3m) 3m ago 3m 70.3M - 17.2.0 e1d6a67b021e 5ae3638f8b1c 2024-04-02T04:54:02.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:01 smithi192 ceph-mon[25697]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:02.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:01 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:54:02.718 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph versions' 2024-04-02T04:54:03.962 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:03 smithi086 ceph-mon[24811]: from='client.14949 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:54:03.962 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:03 smithi086 ceph-mon[24811]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:54:03.963 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:03 smithi086 ceph-mon[29505]: from='client.14949 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:54:03.963 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:03 smithi086 ceph-mon[29505]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:54:04.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:03 smithi192 ceph-mon[25697]: from='client.14949 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:54:04.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:03 smithi192 ceph-mon[25697]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:54:04.539 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T04:54:04.539 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": { 2024-04-02T04:54:04.539 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-02T04:54:04.540 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T04:54:04.540 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": { 2024-04-02T04:54:04.540 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-02T04:54:04.540 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T04:54:04.540 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": { 2024-04-02T04:54:04.540 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-02T04:54:04.540 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T04:54:04.540 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": {}, 2024-04-02T04:54:04.540 INFO:teuthology.orchestra.run.smithi086.stdout: "rgw": { 2024-04-02T04:54:04.540 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-02T04:54:04.540 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T04:54:04.540 INFO:teuthology.orchestra.run.smithi086.stdout: "overall": { 2024-04-02T04:54:04.540 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 15 2024-04-02T04:54:04.540 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-04-02T04:54:04.541 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T04:54:05.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:04 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/690639572' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T04:54:05.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:04 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/690639572' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T04:54:05.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:04 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/690639572' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T04:54:05.299 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph -s' 2024-04-02T04:54:05.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:05 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:54:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:54:05.926 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:05 smithi086 ceph-mon[29505]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:05.927 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:05 smithi086 ceph-mon[24811]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:05 smithi192 ceph-mon[25697]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:07.380 INFO:teuthology.orchestra.run.smithi086.stdout: cluster: 2024-04-02T04:54:07.380 INFO:teuthology.orchestra.run.smithi086.stdout: id: 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 2024-04-02T04:54:07.380 INFO:teuthology.orchestra.run.smithi086.stdout: health: HEALTH_OK 2024-04-02T04:54:07.380 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-02T04:54:07.380 INFO:teuthology.orchestra.run.smithi086.stdout: services: 2024-04-02T04:54:07.380 INFO:teuthology.orchestra.run.smithi086.stdout: mon: 3 daemons, quorum a,c,b (age 8m) 2024-04-02T04:54:07.380 INFO:teuthology.orchestra.run.smithi086.stdout: mgr: y(active, since 5m), standbys: x 2024-04-02T04:54:07.380 INFO:teuthology.orchestra.run.smithi086.stdout: osd: 8 osds: 8 up (since 5m), 8 in (since 5m) 2024-04-02T04:54:07.380 INFO:teuthology.orchestra.run.smithi086.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-04-02T04:54:07.380 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-02T04:54:07.381 INFO:teuthology.orchestra.run.smithi086.stdout: data: 2024-04-02T04:54:07.381 INFO:teuthology.orchestra.run.smithi086.stdout: pools: 6 pools, 161 pgs 2024-04-02T04:54:07.381 INFO:teuthology.orchestra.run.smithi086.stdout: objects: 241 objects, 585 KiB 2024-04-02T04:54:07.381 INFO:teuthology.orchestra.run.smithi086.stdout: usage: 80 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:54:07.381 INFO:teuthology.orchestra.run.smithi086.stdout: pgs: 161 active+clean 2024-04-02T04:54:07.381 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-02T04:54:07.381 INFO:teuthology.orchestra.run.smithi086.stdout: io: 2024-04-02T04:54:07.381 INFO:teuthology.orchestra.run.smithi086.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-04-02T04:54:07.381 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-02T04:54:08.032 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:07 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:54:08.032 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:07 smithi086 ceph-mon[29505]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:08.032 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:07 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1671435055' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-02T04:54:08.032 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:07 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:54:08.032 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:07 smithi086 ceph-mon[24811]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:08.032 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:07 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1671435055' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-02T04:54:08.055 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch ls' 2024-04-02T04:54:08.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:07 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:54:08.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:07 smithi192 ceph-mon[25697]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:08.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:07 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1671435055' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-02T04:54:10.018 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:09 smithi086 ceph-mon[24811]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:54:10.019 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:09 smithi086 ceph-mon[29505]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:54:10.019 INFO:teuthology.orchestra.run.smithi086.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-04-02T04:54:10.019 INFO:teuthology.orchestra.run.smithi086.stdout:alertmanager ?:9093,9094 1/1 3m ago 5m smithi086=a;count:1 2024-04-02T04:54:10.019 INFO:teuthology.orchestra.run.smithi086.stdout:grafana ?:3000 1/1 3m ago 5m smithi192=a;count:1 2024-04-02T04:54:10.019 INFO:teuthology.orchestra.run.smithi086.stdout:iscsi.foo 1/1 3m ago 3m count:1 2024-04-02T04:54:10.019 INFO:teuthology.orchestra.run.smithi086.stdout:mgr 2/2 3m ago 8m smithi086=y;smithi192=x;count:2 2024-04-02T04:54:10.019 INFO:teuthology.orchestra.run.smithi086.stdout:mon 3/3 3m ago 8m smithi086:172.21.15.86=a;smithi086:[v2:172.21.15.86:3301,v1:172.21.15.86:6790]=c;smithi192:172.21.15.192=b;count:3 2024-04-02T04:54:10.019 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter ?:9100 2/2 3m ago 5m smithi086=a;smithi192=b;count:2 2024-04-02T04:54:10.019 INFO:teuthology.orchestra.run.smithi086.stdout:osd 8 3m ago - 2024-04-02T04:54:10.019 INFO:teuthology.orchestra.run.smithi086.stdout:prometheus ?:9095 1/1 3m ago 5m smithi192=a;count:1 2024-04-02T04:54:10.019 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo ?:8000 2/2 3m ago 3m count:2 2024-04-02T04:54:10.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:09 smithi192 ceph-mon[25697]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:54:10.333 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:54:10 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:54:10] "GET /metrics HTTP/1.1" 200 207996 "" "Prometheus/2.33.4" 2024-04-02T04:54:10.583 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-02T04:54:12.001 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:11 smithi086 ceph-mon[29505]: from='client.14967 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:54:12.002 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:11 smithi086 ceph-mon[29505]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:12.002 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:11 smithi086 ceph-mon[24811]: from='client.14967 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:54:12.002 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:11 smithi086 ceph-mon[24811]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:12.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:11 smithi192 ceph-mon[25697]: from='client.14967 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:54:12.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:11 smithi192 ceph-mon[25697]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:12.738 INFO:teuthology.orchestra.run.smithi086.stdout:Scheduled to redeploy mgr.x on host 'smithi192' 2024-04-02T04:54:13.065 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:12 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/2189173135' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-02T04:54:13.066 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:12 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/2189173135' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-02T04:54:13.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:12 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/2189173135' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-02T04:54:13.442 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch ps --refresh' 2024-04-02T04:54:14.037 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:13 smithi086 ceph-mon[29505]: from='client.14979 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:54:14.037 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:13 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:54:14.037 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:13 smithi086 ceph-mon[29505]: Schedule redeploy daemon mgr.x 2024-04-02T04:54:14.037 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:13 smithi086 ceph-mon[29505]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:54:14.037 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:13 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:54:14.037 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:13 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:54:14.037 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:13 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:54:14.037 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:13 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T04:54:14.037 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:13 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T04:54:14.037 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:13 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:54:14.037 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:13 smithi086 ceph-mon[29505]: Deploying daemon mgr.x on smithi192 2024-04-02T04:54:14.038 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:13 smithi086 ceph-mon[24811]: from='client.14979 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:54:14.038 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:13 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:54:14.038 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:13 smithi086 ceph-mon[24811]: Schedule redeploy daemon mgr.x 2024-04-02T04:54:14.038 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:13 smithi086 ceph-mon[24811]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:54:14.038 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:13 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:54:14.038 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:13 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:54:14.038 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:13 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:54:14.038 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:13 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T04:54:14.038 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:13 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T04:54:14.038 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:13 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:54:14.038 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:13 smithi086 ceph-mon[24811]: Deploying daemon mgr.x on smithi192 2024-04-02T04:54:14.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:13 smithi192 ceph-mon[25697]: from='client.14979 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:54:14.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:13 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:54:14.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:13 smithi192 ceph-mon[25697]: Schedule redeploy daemon mgr.x 2024-04-02T04:54:14.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:13 smithi192 ceph-mon[25697]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:54:14.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:13 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:54:14.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:13 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:54:14.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:13 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:54:14.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:13 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T04:54:14.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:13 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T04:54:14.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:13 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:54:14.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:13 smithi192 ceph-mon[25697]: Deploying daemon mgr.x on smithi192 2024-04-02T04:54:15.397 INFO:teuthology.orchestra.run.smithi086.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T04:54:15.397 INFO:teuthology.orchestra.run.smithi086.stdout:alertmanager.a smithi086 *:9093,9094 running (3m) 3m ago 4m 14.0M - ba2b418f427c 24a36455c114 2024-04-02T04:54:15.397 INFO:teuthology.orchestra.run.smithi086.stdout:grafana.a smithi192 *:3000 running (3m) 3m ago 3m 44.8M - 8.3.5 dad864ee21e9 6517aa8b3121 2024-04-02T04:54:15.397 INFO:teuthology.orchestra.run.smithi086.stdout:iscsi.foo.smithi086.rsbqwx smithi086 running (3m) 3m ago 3m 66.8M - 3.5 e1d6a67b021e 828835e36fd4 2024-04-02T04:54:15.397 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.x smithi192 *:8443 running (8m) 3m ago 8m 410M - 17.2.0 e1d6a67b021e 2ebae22b0016 2024-04-02T04:54:15.397 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.y smithi086 *:9283 running (10m) 3m ago 10m 453M - 17.2.0 e1d6a67b021e 02f336a19fd6 2024-04-02T04:54:15.398 INFO:teuthology.orchestra.run.smithi086.stdout:mon.a smithi086 running (10m) 3m ago 10m 46.6M 2048M 17.2.0 e1d6a67b021e b6c507ab96f8 2024-04-02T04:54:15.398 INFO:teuthology.orchestra.run.smithi086.stdout:mon.b smithi192 running (8m) 3m ago 8m 40.9M 2048M 17.2.0 e1d6a67b021e 4dadd8d1c318 2024-04-02T04:54:15.398 INFO:teuthology.orchestra.run.smithi086.stdout:mon.c smithi086 running (9m) 3m ago 9m 38.0M 2048M 17.2.0 e1d6a67b021e 49275483a2d4 2024-04-02T04:54:15.398 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.a smithi086 *:9100 running (4m) 3m ago 4m 11.1M - 1dbe0e931976 0151f4e72bd9 2024-04-02T04:54:15.398 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.b smithi192 *:9100 running (4m) 3m ago 4m 15.1M - 1dbe0e931976 27476f65f84f 2024-04-02T04:54:15.398 INFO:teuthology.orchestra.run.smithi086.stdout:osd.0 smithi086 running (8m) 3m ago 8m 51.6M 4016M 17.2.0 e1d6a67b021e 6d90c60d1c71 2024-04-02T04:54:15.398 INFO:teuthology.orchestra.run.smithi086.stdout:osd.1 smithi086 running (7m) 3m ago 7m 49.7M 4016M 17.2.0 e1d6a67b021e 203689f71603 2024-04-02T04:54:15.398 INFO:teuthology.orchestra.run.smithi086.stdout:osd.2 smithi086 running (7m) 3m ago 7m 43.8M 4016M 17.2.0 e1d6a67b021e 472c860a9af8 2024-04-02T04:54:15.398 INFO:teuthology.orchestra.run.smithi086.stdout:osd.3 smithi086 running (7m) 3m ago 7m 45.2M 4016M 17.2.0 e1d6a67b021e 327538dac44e 2024-04-02T04:54:15.398 INFO:teuthology.orchestra.run.smithi086.stdout:osd.4 smithi192 running (6m) 3m ago 6m 45.9M 4272M 17.2.0 e1d6a67b021e 6535d7919a2d 2024-04-02T04:54:15.398 INFO:teuthology.orchestra.run.smithi086.stdout:osd.5 smithi192 running (6m) 3m ago 6m 46.6M 4272M 17.2.0 e1d6a67b021e 7259249605f2 2024-04-02T04:54:15.398 INFO:teuthology.orchestra.run.smithi086.stdout:osd.6 smithi192 running (6m) 3m ago 6m 44.2M 4272M 17.2.0 e1d6a67b021e bb7d171b02a1 2024-04-02T04:54:15.398 INFO:teuthology.orchestra.run.smithi086.stdout:osd.7 smithi192 running (5m) 3m ago 5m 46.1M 4272M 17.2.0 e1d6a67b021e 95f5a0353999 2024-04-02T04:54:15.398 INFO:teuthology.orchestra.run.smithi086.stdout:prometheus.a smithi192 *:9095 running (3m) 3m ago 5m 44.5M - 514e6a882f6e 782ae1d5e7f9 2024-04-02T04:54:15.399 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi086.bxzxlp smithi086 *:8000 running (3m) 3m ago 3m 71.5M - 17.2.0 e1d6a67b021e 6725785fbc53 2024-04-02T04:54:15.399 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi192.pmlbot smithi192 *:8000 running (3m) 3m ago 3m 70.3M - 17.2.0 e1d6a67b021e 5ae3638f8b1c 2024-04-02T04:54:15.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:15 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:54:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:54:16.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:15 smithi086 ceph-mon[24811]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:16.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:15 smithi086 ceph-mon[29505]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:16.162 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'sleep 180' 2024-04-02T04:54:16.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:15 smithi192 ceph-mon[25697]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:17.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:16 smithi086 ceph-mon[24811]: from='client.14985 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:54:17.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:16 smithi086 ceph-mon[29505]: from='client.14985 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:54:17.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:16 smithi192 ceph-mon[25697]: from='client.14985 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:54:18.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:17 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:54:18.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:17 smithi086 ceph-mon[24811]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:18.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:17 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:54:18.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:17 smithi086 ceph-mon[29505]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:18.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:17 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:54:18.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:17 smithi192 ceph-mon[25697]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:20.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:19 smithi086 ceph-mon[24811]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:54:20.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:19 smithi086 ceph-mon[29505]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:54:20.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:19 smithi192 ceph-mon[25697]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:54:20.584 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:54:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:54:20] "GET /metrics HTTP/1.1" 200 207992 "" "Prometheus/2.33.4" 2024-04-02T04:54:22.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:21 smithi086 ceph-mon[24811]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:22.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:21 smithi086 ceph-mon[29505]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:22.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:21 smithi192 ceph-mon[25697]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:24.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:23 smithi086 ceph-mon[29505]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:54:24.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:23 smithi086 ceph-mon[24811]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:54:24.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:23 smithi192 ceph-mon[25697]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:54:25.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:25 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:54:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:54:26.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:25 smithi086 ceph-mon[24811]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:26.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:25 smithi086 ceph-mon[29505]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:26.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:25 smithi192 ceph-mon[25697]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:28.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:27 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:54:28.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:27 smithi086 ceph-mon[24811]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:28.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:27 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:54:28.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:27 smithi086 ceph-mon[29505]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:27 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:54:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:27 smithi192 ceph-mon[25697]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:30.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:29 smithi086 ceph-mon[29505]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:54:30.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:29 smithi086 ceph-mon[24811]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:54:30.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:29 smithi192 ceph-mon[25697]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:54:30.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:54:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:54:30] "GET /metrics HTTP/1.1" 200 207982 "" "Prometheus/2.33.4" 2024-04-02T04:54:32.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:31 smithi086 ceph-mon[29505]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:32.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:31 smithi086 ceph-mon[24811]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:32.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:31 smithi192 ceph-mon[25697]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:34.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:33 smithi086 ceph-mon[24811]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:54:34.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:33 smithi086 ceph-mon[29505]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:54:34.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:33 smithi192 ceph-mon[25697]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:54:35.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:35 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:54:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:54:36.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:35 smithi086 ceph-mon[24811]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:36.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:35 smithi086 ceph-mon[29505]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:36.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:35 smithi192 ceph-mon[25697]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:38.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:37 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:54:38.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:37 smithi086 ceph-mon[24811]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:38.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:37 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:54:38.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:37 smithi086 ceph-mon[29505]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:38.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:37 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:54:38.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:37 smithi192 ceph-mon[25697]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:40.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:39 smithi086 ceph-mon[24811]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:54:40.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:39 smithi086 ceph-mon[29505]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:54:40.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:39 smithi192 ceph-mon[25697]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:54:40.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:54:40 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:54:40] "GET /metrics HTTP/1.1" 200 207982 "" "Prometheus/2.33.4" 2024-04-02T04:54:42.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:41 smithi086 ceph-mon[24811]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:42.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:41 smithi086 ceph-mon[29505]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:42.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:41 smithi192 ceph-mon[25697]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:44.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:43 smithi086 ceph-mon[24811]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:54:44.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:43 smithi086 ceph-mon[29505]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:54:44.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:43 smithi192 ceph-mon[25697]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:54:45.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:45 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[26881]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:54:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:54:45.999 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:45 smithi192 ceph-mon[25697]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:46.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:45 smithi086 ceph-mon[24811]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:46.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:45 smithi086 ceph-mon[29505]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:47.094 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:46 smithi192 systemd[1]: Stopping Ceph mgr.x for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T04:54:47.094 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:46 smithi192 bash[45264]: Error: no container with name or ID "ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr.x" found: no such container 2024-04-02T04:54:47.407 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:47 smithi192 podman[45277]: 2024-04-02 04:54:47.091075754 +0000 UTC m=+0.121232714 container died 2ebae22b00160c51becaf9945e219459ffdd09228a47cf925078b28b94209bd0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, name=centos-stream, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, vcs-type=git, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD) 2024-04-02T04:54:47.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:47 smithi192 podman[45277]: 2024-04-02 04:54:47.478987002 +0000 UTC m=+0.509143968 container cleanup 2ebae22b00160c51becaf9945e219459ffdd09228a47cf925078b28b94209bd0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, version=8, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, GIT_BRANCH=HEAD, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, ceph=True, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, release=754) 2024-04-02T04:54:47.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:47 smithi192 bash[45277]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x 2024-04-02T04:54:47.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:47 smithi192 bash[45305]: Error: no container with name or ID "ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr.x" found: no such container 2024-04-02T04:54:48.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:47 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:54:48.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:47 smithi086 ceph-mon[24811]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:48.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:47 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:54:48.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:47 smithi086 ceph-mon[29505]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:48.110 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:47 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:54:48.110 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:47 smithi192 ceph-mon[25697]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:48.110 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:47 smithi192 podman[45292]: 2024-04-02 04:54:47.69698389 +0000 UTC m=+0.598137134 container remove 2ebae22b00160c51becaf9945e219459ffdd09228a47cf925078b28b94209bd0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x, release=754, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, architecture=x86_64, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, io.buildah.version=1.19.8, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, vendor=Red Hat, Inc., RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-02T04:54:48.110 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:47 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-02T04:54:48.110 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:47 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mgr.x.service: Failed with result 'exit-code'. 2024-04-02T04:54:48.110 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:47 smithi192 systemd[1]: Stopped Ceph mgr.x for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T04:54:48.110 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:47 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mgr.x.service: Consumed 16.746s CPU time. 2024-04-02T04:54:48.110 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:47 smithi192 systemd[1]: Starting Ceph mgr.x for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T04:54:48.412 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:48 smithi192 podman[45411]: 2024-04-02 04:54:48.106697101 +0000 UTC m=+0.113007463 container create fd618149d6b41b92546c4791f5c712a6d3d78106a1d99348c3be723483b74eaa (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, RELEASE=squid-6cec793, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-04-02T04:54:48.412 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:48 smithi192 podman[45411]: 2024-04-02 04:54:48.011850107 +0000 UTC m=+0.018160493 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T04:54:48.412 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:48 smithi192 podman[45411]: 2024-04-02 04:54:48.325779802 +0000 UTC m=+0.332090165 container init fd618149d6b41b92546c4791f5c712a6d3d78106a1d99348c3be723483b74eaa (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-6cec793, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-02T04:54:48.412 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:48 smithi192 podman[45411]: 2024-04-02 04:54:48.328444645 +0000 UTC m=+0.334755018 container start fd618149d6b41b92546c4791f5c712a6d3d78106a1d99348c3be723483b74eaa (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, RELEASE=squid-6cec793, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True) 2024-04-02T04:54:48.413 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:48 smithi192 ceph-mgr[45429]: -- 172.21.15.192:0/2422486542 <== mon.2 v2:172.21.15.192:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f1517765a0 con 0x55f15174a800 2024-04-02T04:54:48.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:48 smithi192 bash[45411]: fd618149d6b41b92546c4791f5c712a6d3d78106a1d99348c3be723483b74eaa 2024-04-02T04:54:48.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:48 smithi192 systemd[1]: Started Ceph mgr.x for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T04:54:49.178 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:48 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T04:54:48.910+0000 7f9e945c7200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-02T04:54:49.178 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:49 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T04:54:49.004+0000 7f9e945c7200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-02T04:54:49.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:49 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:54:49.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:49 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:54:49.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:49 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:54:49.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:49 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:54:49.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:49 smithi086 ceph-mon[24811]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:54:49.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:49 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:54:49.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:49 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:54:49.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:49 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:54:49.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:49 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:54:49.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:49 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:54:49.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:49 smithi086 ceph-mon[29505]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:54:49.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:49 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:54:49.761 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:49 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T04:54:49.691+0000 7f9e945c7200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-02T04:54:49.761 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:49 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:54:49.761 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:49 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:54:49.761 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:49 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:54:49.761 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:49 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:54:49.761 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:49 smithi192 ceph-mon[25697]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:54:49.761 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:49 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:54:50.056 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:49 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T04:54:49.797+0000 7f9e945c7200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-02T04:54:50.056 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:49 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T04:54:49.946+0000 7f9e945c7200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-02T04:54:50.403 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:54:50 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:54:50] "GET /metrics HTTP/1.1" 200 207985 "" "Prometheus/2.33.4" 2024-04-02T04:54:50.428 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:50 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T04:54:50.052+0000 7f9e945c7200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-02T04:54:50.428 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:50 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T04:54:50.155+0000 7f9e945c7200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-02T04:54:50.815 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:50 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T04:54:50.509+0000 7f9e945c7200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-02T04:54:50.815 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:50 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T04:54:50.624+0000 7f9e945c7200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-02T04:54:50.815 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:50 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T04:54:50.712+0000 7f9e945c7200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-02T04:54:50.815 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:50 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T04:54:50.811+0000 7f9e945c7200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-02T04:54:51.178 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:51 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T04:54:51.008+0000 7f9e945c7200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-02T04:54:52.020 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:51 smithi192 ceph-mon[25697]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:52.024 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:51 smithi086 ceph-mon[29505]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:52.024 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:51 smithi086 ceph-mon[24811]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:52.354 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:52 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T04:54:52.038+0000 7f9e945c7200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-02T04:54:52.354 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:52 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T04:54:52.171+0000 7f9e945c7200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-02T04:54:52.614 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:52 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T04:54:52.519+0000 7f9e945c7200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-02T04:54:52.614 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:52 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T04:54:52.610+0000 7f9e945c7200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-02T04:54:52.928 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:52 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T04:54:52.808+0000 7f9e945c7200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-02T04:54:53.285 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:52 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T04:54:52.996+0000 7f9e945c7200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-02T04:54:53.286 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:53 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:54:53.286 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:53 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:54:53.286 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:53 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:54:53.286 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:53 smithi192 ceph-mon[25697]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:54:53.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:53 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:54:53.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:53 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:54:53.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:53 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:54:53.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:53 smithi086 ceph-mon[24811]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:54:53.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:53 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:54:53.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:53 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:54:53.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:53 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:54:53.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:53 smithi086 ceph-mon[29505]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:54:53.549 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:53 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T04:54:53.281+0000 7f9e945c7200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-02T04:54:53.928 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:53 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T04:54:53.545+0000 7f9e945c7200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-02T04:54:53.928 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:53 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T04:54:53.783+0000 7f9e945c7200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-02T04:54:54.928 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:54 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T04:54:54.587+0000 7f9e945c7200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-02T04:54:55.428 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:55 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T04:54:55.163+0000 7f9e945c7200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-02T04:54:55.428 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:55 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T04:54:55.265+0000 7f9e945c7200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-02T04:54:55.806 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:55 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T04:54:55.484+0000 7f9e945c7200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-02T04:54:55.806 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:55 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:04:54:55] ENGINE Bus STARTING 2024-04-02T04:54:55.806 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:55 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: CherryPy Checker: 2024-04-02T04:54:55.806 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:55 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: The Application mounted at '' has an empty config. 2024-04-02T04:54:55.806 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:55 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T04:54:55.807 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:55 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:04:54:55] ENGINE Serving on http://:::9283 2024-04-02T04:54:55.807 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:54:55 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:04:54:55] ENGINE Bus STARTED 2024-04-02T04:54:56.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:55 smithi086 ceph-mon[24811]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:56.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:55 smithi086 ceph-mon[24811]: Standby manager daemon x restarted 2024-04-02T04:54:56.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:55 smithi086 ceph-mon[24811]: Standby manager daemon x started 2024-04-02T04:54:56.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:55 smithi086 ceph-mon[24811]: from='mgr.? 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-02T04:54:56.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:55 smithi086 ceph-mon[24811]: from='mgr.? 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T04:54:56.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:55 smithi086 ceph-mon[24811]: from='mgr.? 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-02T04:54:56.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:55 smithi086 ceph-mon[24811]: from='mgr.? 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T04:54:56.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:55 smithi086 ceph-mon[29505]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:56.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:55 smithi086 ceph-mon[29505]: Standby manager daemon x restarted 2024-04-02T04:54:56.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:55 smithi086 ceph-mon[29505]: Standby manager daemon x started 2024-04-02T04:54:56.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:55 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-02T04:54:56.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:55 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T04:54:56.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:55 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-02T04:54:56.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:55 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T04:54:56.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:55 smithi192 ceph-mon[25697]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:56.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:55 smithi192 ceph-mon[25697]: Standby manager daemon x restarted 2024-04-02T04:54:56.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:55 smithi192 ceph-mon[25697]: Standby manager daemon x started 2024-04-02T04:54:56.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:55 smithi192 ceph-mon[25697]: from='mgr.? 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-02T04:54:56.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:55 smithi192 ceph-mon[25697]: from='mgr.? 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T04:54:56.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:55 smithi192 ceph-mon[25697]: from='mgr.? 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-02T04:54:56.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:55 smithi192 ceph-mon[25697]: from='mgr.? 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T04:54:57.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:56 smithi086 ceph-mon[24811]: mgrmap e21: y(active, since 6m), standbys: x 2024-04-02T04:54:57.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:56 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:54:57.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:56 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:54:57.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:56 smithi086 ceph-mon[29505]: mgrmap e21: y(active, since 6m), standbys: x 2024-04-02T04:54:57.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:56 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:54:57.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:56 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:54:57.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:56 smithi192 ceph-mon[25697]: mgrmap e21: y(active, since 6m), standbys: x 2024-04-02T04:54:57.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:56 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:54:57.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:56 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:54:58.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:57 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:54:58.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:57 smithi086 ceph-mon[24811]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:58.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:57 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:54:58.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:57 smithi086 ceph-mon[29505]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:54:58.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:57 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:54:58.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:57 smithi192 ceph-mon[25697]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:00.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:54:59 smithi086 ceph-mon[24811]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:55:00.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:54:59 smithi086 ceph-mon[29505]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:55:00.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:54:59 smithi192 ceph-mon[25697]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:55:00.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:55:00 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:55:00] "GET /metrics HTTP/1.1" 200 208003 "" "Prometheus/2.33.4" 2024-04-02T04:55:02.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:01 smithi086 ceph-mon[24811]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:02.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:01 smithi086 ceph-mon[29505]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:02.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:01 smithi192 ceph-mon[25697]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:03.807 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:55:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:55:03.508Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.86:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.86 because it doesn't contain any IP SANs" 2024-04-02T04:55:03.807 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:55:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:55:03.508Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.192:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.192 because it doesn't contain any IP SANs" 2024-04-02T04:55:04.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:03 smithi086 ceph-mon[24811]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:55:04.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:03 smithi086 ceph-mon[29505]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:55:04.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:03 smithi192 ceph-mon[25697]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:55:05.480 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:55:05 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:55:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:55:06.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:05 smithi086 ceph-mon[24811]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:06.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:05 smithi086 ceph-mon[29505]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:05 smithi192 ceph-mon[25697]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:08.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:07 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:55:08.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:07 smithi086 ceph-mon[24811]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:08.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:07 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:55:08.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:07 smithi086 ceph-mon[29505]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:08.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:07 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:55:08.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:07 smithi192 ceph-mon[25697]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:10.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:09 smithi086 ceph-mon[24811]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:55:10.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:09 smithi086 ceph-mon[29505]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:55:10.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:09 smithi192 ceph-mon[25697]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:55:10.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:55:10 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:55:10] "GET /metrics HTTP/1.1" 200 208003 "" "Prometheus/2.33.4" 2024-04-02T04:55:12.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:11 smithi086 ceph-mon[29505]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:12.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:11 smithi086 ceph-mon[24811]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:12.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:11 smithi192 ceph-mon[25697]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:13.811 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:55:13 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=error ts=2024-04-02T04:55:13.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.86:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.86 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.192:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.192 because it doesn't contain any IP SANs" 2024-04-02T04:55:13.811 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:55:13 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:55:13.508Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.86:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.86 because it doesn't contain any IP SANs" 2024-04-02T04:55:13.811 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:55:13 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:55:13.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.192:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.192 because it doesn't contain any IP SANs" 2024-04-02T04:55:14.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:13 smithi086 ceph-mon[24811]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:55:14.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:13 smithi086 ceph-mon[29505]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:55:14.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:13 smithi192 ceph-mon[25697]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:55:15.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:55:15 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:55:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:55:16.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:15 smithi086 ceph-mon[24811]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:16.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:15 smithi086 ceph-mon[29505]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:16.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:15 smithi192 ceph-mon[25697]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:18.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:17 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:55:18.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:17 smithi086 ceph-mon[24811]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:18.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:17 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:55:18.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:17 smithi086 ceph-mon[29505]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:18.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:17 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:55:18.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:17 smithi192 ceph-mon[25697]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:20.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:19 smithi086 ceph-mon[24811]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:55:20.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:19 smithi086 ceph-mon[29505]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:55:20.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:19 smithi192 ceph-mon[25697]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:55:20.584 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:55:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:55:20] "GET /metrics HTTP/1.1" 200 208009 "" "Prometheus/2.33.4" 2024-04-02T04:55:22.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:21 smithi086 ceph-mon[29505]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:22.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:21 smithi086 ceph-mon[24811]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:22.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:21 smithi192 ceph-mon[25697]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:23.813 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:55:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=error ts=2024-04-02T04:55:23.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.86:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.86 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.192:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.192 because it doesn't contain any IP SANs" 2024-04-02T04:55:23.814 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:55:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:55:23.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.86:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.86 because it doesn't contain any IP SANs" 2024-04-02T04:55:23.814 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:55:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:55:23.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.192:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.192 because it doesn't contain any IP SANs" 2024-04-02T04:55:24.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:23 smithi086 ceph-mon[24811]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:55:24.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:23 smithi086 ceph-mon[29505]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:55:24.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:23 smithi192 ceph-mon[25697]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:55:25.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:55:25 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:55:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:55:26.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:25 smithi086 ceph-mon[24811]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:26.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:25 smithi086 ceph-mon[29505]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:26.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:25 smithi192 ceph-mon[25697]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:28.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:27 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:55:28.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:27 smithi086 ceph-mon[29505]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:28.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:27 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:55:28.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:27 smithi086 ceph-mon[24811]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:27 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:55:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:27 smithi192 ceph-mon[25697]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:30.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:29 smithi086 ceph-mon[29505]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:55:30.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:29 smithi086 ceph-mon[24811]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:55:30.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:29 smithi192 ceph-mon[25697]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:55:30.584 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:55:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:55:30] "GET /metrics HTTP/1.1" 200 208002 "" "Prometheus/2.33.4" 2024-04-02T04:55:32.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:31 smithi086 ceph-mon[24811]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:32.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:31 smithi086 ceph-mon[29505]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:32.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:31 smithi192 ceph-mon[25697]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:33.816 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:55:33 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=error ts=2024-04-02T04:55:33.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.192:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.192 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.86:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.86 because it doesn't contain any IP SANs" 2024-04-02T04:55:33.816 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:55:33 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:55:33.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.86:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.86 because it doesn't contain any IP SANs" 2024-04-02T04:55:33.817 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:55:33 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:55:33.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.192:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.192 because it doesn't contain any IP SANs" 2024-04-02T04:55:34.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:33 smithi086 ceph-mon[24811]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:55:34.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:33 smithi086 ceph-mon[29505]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:55:34.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:33 smithi192 ceph-mon[25697]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:55:35.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:55:35 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:55:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:55:36.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:35 smithi086 ceph-mon[24811]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:36.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:35 smithi086 ceph-mon[29505]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:36.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:35 smithi192 ceph-mon[25697]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:38.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:37 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:55:38.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:37 smithi086 ceph-mon[24811]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:38.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:37 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:55:38.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:37 smithi086 ceph-mon[29505]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:38.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:37 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:55:38.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:37 smithi192 ceph-mon[25697]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:40.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:39 smithi086 ceph-mon[24811]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:55:40.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:39 smithi086 ceph-mon[29505]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:55:40.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:39 smithi192 ceph-mon[25697]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:55:40.584 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:55:40 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:55:40] "GET /metrics HTTP/1.1" 200 208002 "" "Prometheus/2.33.4" 2024-04-02T04:55:42.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:41 smithi086 ceph-mon[29505]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-02T04:55:42.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:41 smithi086 ceph-mon[24811]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-02T04:55:42.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:41 smithi192 ceph-mon[25697]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-02T04:55:43.821 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:55:43 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=error ts=2024-04-02T04:55:43.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.192:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.192 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.86:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.86 because it doesn't contain any IP SANs" 2024-04-02T04:55:43.821 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:55:43 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:55:43.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.86:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.86 because it doesn't contain any IP SANs" 2024-04-02T04:55:43.821 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:55:43 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:55:43.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.192:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.192 because it doesn't contain any IP SANs" 2024-04-02T04:55:44.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:43 smithi086 ceph-mon[24811]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:55:44.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:43 smithi086 ceph-mon[29505]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:55:44.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:43 smithi192 ceph-mon[25697]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:55:45.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:55:45 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:55:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:55:46.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:45 smithi086 ceph-mon[24811]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:46.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:45 smithi086 ceph-mon[29505]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:46.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:45 smithi192 ceph-mon[25697]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:48.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:47 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:55:48.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:47 smithi086 ceph-mon[24811]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:48.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:47 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:55:48.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:47 smithi086 ceph-mon[29505]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:48.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:47 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:55:48.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:47 smithi192 ceph-mon[25697]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:50.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:49 smithi086 ceph-mon[24811]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:55:50.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:49 smithi086 ceph-mon[29505]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:55:50.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:49 smithi192 ceph-mon[25697]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:55:50.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:55:50 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:55:50] "GET /metrics HTTP/1.1" 200 208010 "" "Prometheus/2.33.4" 2024-04-02T04:55:52.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:51 smithi086 ceph-mon[24811]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:52.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:51 smithi086 ceph-mon[29505]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:52.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:51 smithi192 ceph-mon[25697]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:53.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:52 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:55:53.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:52 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:55:53.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:52 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:55:53.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:52 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:55:53.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:52 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:55:53.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:52 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:55:53.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:52 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:55:53.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:52 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:55:53.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:52 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:55:53.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:53 smithi086 ceph-mon[24811]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:55:53.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:53 smithi086 ceph-mon[29505]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:55:53.834 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:55:53 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=error ts=2024-04-02T04:55:53.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.192:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.192 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.86:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.86 because it doesn't contain any IP SANs" 2024-04-02T04:55:53.834 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:55:53 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:55:53.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.192:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.192 because it doesn't contain any IP SANs" 2024-04-02T04:55:53.834 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:55:53 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:55:53.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.86:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.86 because it doesn't contain any IP SANs" 2024-04-02T04:55:54.074 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:53 smithi192 ceph-mon[25697]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:55:55.449 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:55:55 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:55:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:55:56.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:55 smithi086 ceph-mon[24811]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:56.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:55 smithi086 ceph-mon[29505]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:56.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:55 smithi192 ceph-mon[25697]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:58.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:57 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:55:58.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:57 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:55:58.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:57 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:55:58.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:57 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:55:58.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:57 smithi086 ceph-mon[24811]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:58.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:57 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:55:58.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:57 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:55:58.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:57 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:55:58.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:57 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:55:58.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:57 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:55:58.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:57 smithi086 ceph-mon[29505]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:58.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:57 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:55:58.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:57 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:55:58.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:57 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:55:58.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:57 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:55:58.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:57 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:55:58.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:57 smithi192 ceph-mon[25697]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:55:58.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:57 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:56:00.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:55:59 smithi086 ceph-mon[24811]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:56:00.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:55:59 smithi086 ceph-mon[29505]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:56:00.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:55:59 smithi192 ceph-mon[25697]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:56:00.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:56:00 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:56:00] "GET /metrics HTTP/1.1" 200 208000 "" "Prometheus/2.33.4" 2024-04-02T04:56:02.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:01 smithi086 ceph-mon[24811]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:02.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:01 smithi086 ceph-mon[29505]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:02.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:01 smithi192 ceph-mon[25697]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:03.826 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:56:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=error ts=2024-04-02T04:56:03.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.86:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.86 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.192:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.192 because it doesn't contain any IP SANs" 2024-04-02T04:56:03.827 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:56:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:56:03.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.86:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.86 because it doesn't contain any IP SANs" 2024-04-02T04:56:03.827 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:56:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:56:03.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.192:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.192 because it doesn't contain any IP SANs" 2024-04-02T04:56:04.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:03 smithi086 ceph-mon[24811]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:56:04.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:03 smithi086 ceph-mon[29505]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:56:04.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:03 smithi192 ceph-mon[25697]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:56:05.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:56:05 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:56:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:56:06.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:05 smithi086 ceph-mon[24811]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:06.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:05 smithi086 ceph-mon[29505]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:05 smithi192 ceph-mon[25697]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:08.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:07 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:56:08.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:07 smithi086 ceph-mon[29505]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:08.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:07 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:56:08.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:07 smithi086 ceph-mon[24811]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:08.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:07 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:56:08.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:07 smithi192 ceph-mon[25697]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:10.130 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:09 smithi086 ceph-mon[29505]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:56:10.130 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:09 smithi086 ceph-mon[24811]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:56:10.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:09 smithi192 ceph-mon[25697]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:56:10.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:56:10 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:56:10] "GET /metrics HTTP/1.1" 200 208000 "" "Prometheus/2.33.4" 2024-04-02T04:56:12.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:11 smithi086 ceph-mon[24811]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:12.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:11 smithi086 ceph-mon[29505]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:12.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:11 smithi192 ceph-mon[25697]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:13.829 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:56:13 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=error ts=2024-04-02T04:56:13.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.86:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.86 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.192:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.192 because it doesn't contain any IP SANs" 2024-04-02T04:56:13.830 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:56:13 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:56:13.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.86:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.86 because it doesn't contain any IP SANs" 2024-04-02T04:56:13.830 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:56:13 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:56:13.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.192:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.192 because it doesn't contain any IP SANs" 2024-04-02T04:56:14.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:13 smithi086 ceph-mon[24811]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:56:14.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:13 smithi086 ceph-mon[29505]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:56:14.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:13 smithi192 ceph-mon[25697]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:56:15.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:56:15 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:56:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:56:16.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:15 smithi192 ceph-mon[25697]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:16.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:15 smithi086 ceph-mon[29505]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:16.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:15 smithi086 ceph-mon[24811]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:18.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:17 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:56:18.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:17 smithi192 ceph-mon[25697]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:18.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:17 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:56:18.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:17 smithi086 ceph-mon[29505]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:18.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:17 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:56:18.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:17 smithi086 ceph-mon[24811]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:20.130 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:19 smithi086 ceph-mon[24811]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:56:20.130 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:19 smithi086 ceph-mon[29505]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:56:20.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:19 smithi192 ceph-mon[25697]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:56:20.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:56:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:56:20] "GET /metrics HTTP/1.1" 200 208007 "" "Prometheus/2.33.4" 2024-04-02T04:56:22.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:21 smithi086 ceph-mon[24811]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:22.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:21 smithi086 ceph-mon[29505]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:22.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:21 smithi192 ceph-mon[25697]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:23.832 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:56:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=error ts=2024-04-02T04:56:23.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.192:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.192 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.86:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.86 because it doesn't contain any IP SANs" 2024-04-02T04:56:23.832 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:56:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:56:23.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.86:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.86 because it doesn't contain any IP SANs" 2024-04-02T04:56:23.832 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:56:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:56:23.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.192:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.192 because it doesn't contain any IP SANs" 2024-04-02T04:56:24.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:23 smithi086 ceph-mon[24811]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:56:24.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:23 smithi086 ceph-mon[29505]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:56:24.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:23 smithi192 ceph-mon[25697]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:56:25.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:56:25 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:56:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:56:25.932 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:25 smithi192 ceph-mon[25697]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:26.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:25 smithi086 ceph-mon[24811]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:26.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:25 smithi086 ceph-mon[29505]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:27 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:56:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:27 smithi192 ceph-mon[25697]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:28.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:27 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:56:28.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:27 smithi086 ceph-mon[24811]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:28.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:27 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:56:28.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:27 smithi086 ceph-mon[29505]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:30.130 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:29 smithi086 ceph-mon[29505]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:56:30.130 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:29 smithi086 ceph-mon[24811]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:56:30.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:29 smithi192 ceph-mon[25697]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:56:30.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:56:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:56:30] "GET /metrics HTTP/1.1" 200 208006 "" "Prometheus/2.33.4" 2024-04-02T04:56:32.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:31 smithi086 ceph-mon[24811]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:32.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:31 smithi086 ceph-mon[29505]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:32.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:31 smithi192 ceph-mon[25697]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:33.834 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:56:33 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=error ts=2024-04-02T04:56:33.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.86:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.86 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.192:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.192 because it doesn't contain any IP SANs" 2024-04-02T04:56:33.834 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:56:33 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:56:33.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.86:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.86 because it doesn't contain any IP SANs" 2024-04-02T04:56:33.834 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:56:33 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:56:33.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.192:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.192 because it doesn't contain any IP SANs" 2024-04-02T04:56:34.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:33 smithi192 ceph-mon[25697]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:56:34.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:33 smithi086 ceph-mon[24811]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:56:34.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:33 smithi086 ceph-mon[29505]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:56:35.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:56:35 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:56:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:56:36.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:35 smithi192 ceph-mon[25697]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:36.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:35 smithi086 ceph-mon[24811]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:36.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:35 smithi086 ceph-mon[29505]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:38.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:37 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:56:38.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:37 smithi192 ceph-mon[25697]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:38.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:37 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:56:38.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:37 smithi086 ceph-mon[24811]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:38.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:37 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:56:38.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:37 smithi086 ceph-mon[29505]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:40.130 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:39 smithi086 ceph-mon[24811]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:56:40.131 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:39 smithi086 ceph-mon[29505]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:56:40.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:39 smithi192 ceph-mon[25697]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:56:40.584 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:56:40 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:56:40] "GET /metrics HTTP/1.1" 200 208006 "" "Prometheus/2.33.4" 2024-04-02T04:56:42.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:41 smithi086 ceph-mon[24811]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:42.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:41 smithi086 ceph-mon[29505]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:42.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:41 smithi192 ceph-mon[25697]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:43.834 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:56:43 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=error ts=2024-04-02T04:56:43.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.192:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.192 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.86:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.86 because it doesn't contain any IP SANs" 2024-04-02T04:56:43.834 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:56:43 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:56:43.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.86:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.86 because it doesn't contain any IP SANs" 2024-04-02T04:56:43.834 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:56:43 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:56:43.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.192:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.192 because it doesn't contain any IP SANs" 2024-04-02T04:56:44.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:43 smithi192 ceph-mon[25697]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:56:44.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:43 smithi086 ceph-mon[29505]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:56:44.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:43 smithi086 ceph-mon[24811]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:56:45.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:56:45 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:56:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:56:46.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:45 smithi192 ceph-mon[25697]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:46.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:45 smithi086 ceph-mon[24811]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:46.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:45 smithi086 ceph-mon[29505]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:48.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:47 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:56:48.182 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:47 smithi192 ceph-mon[25697]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:48.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:47 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:56:48.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:47 smithi086 ceph-mon[24811]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:48.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:47 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:56:48.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:47 smithi086 ceph-mon[29505]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:50.129 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:49 smithi086 ceph-mon[29505]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:56:50.130 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:49 smithi086 ceph-mon[24811]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:56:50.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:49 smithi192 ceph-mon[25697]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:56:50.584 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:56:50 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:56:50] "GET /metrics HTTP/1.1" 200 208011 "" "Prometheus/2.33.4" 2024-04-02T04:56:52.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:51 smithi086 ceph-mon[29505]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:52.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:51 smithi086 ceph-mon[24811]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:52.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:51 smithi192 ceph-mon[25697]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:53.834 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:56:53 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=error ts=2024-04-02T04:56:53.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.192:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.192 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.86:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.86 because it doesn't contain any IP SANs" 2024-04-02T04:56:53.834 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:56:53 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:56:53.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.86:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.86 because it doesn't contain any IP SANs" 2024-04-02T04:56:53.834 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:56:53 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:56:53.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.192:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.192 because it doesn't contain any IP SANs" 2024-04-02T04:56:54.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:53 smithi192 ceph-mon[25697]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:56:54.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:53 smithi086 ceph-mon[29505]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:56:54.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:53 smithi086 ceph-mon[24811]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:56:55.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:56:55 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:56:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:56:56.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:55 smithi192 ceph-mon[25697]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:56.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:55 smithi086 ceph-mon[24811]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:56.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:55 smithi086 ceph-mon[29505]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:57.110 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:56 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:56:57.110 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:56 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:56:57.110 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:56 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:56:57.110 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:56 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:56:57.115 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:56 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:56:57.115 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:56 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T04:56:58.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:57 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:56:58.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:57 smithi192 ceph-mon[25697]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:58.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:57 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:56:58.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:57 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:56:58.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:57 smithi192 ceph-mon[25697]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:56:58.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:57 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:56:58.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:57 smithi086 ceph-mon[24811]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:58.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:57 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:56:58.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:57 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:56:58.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:57 smithi086 ceph-mon[24811]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:56:58.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:57 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:56:58.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:57 smithi086 ceph-mon[29505]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:56:58.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:57 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:56:58.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:57 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:56:58.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:57 smithi086 ceph-mon[29505]: from='mgr.14424 172.21.15.86:0/3215665122' entity='mgr.y' 2024-04-02T04:57:00.130 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:56:59 smithi086 ceph-mon[24811]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:57:00.130 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:56:59 smithi086 ceph-mon[29505]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:57:00.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:56:59 smithi192 ceph-mon[25697]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:57:00.584 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:00 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:57:00] "GET /metrics HTTP/1.1" 200 208013 "" "Prometheus/2.33.4" 2024-04-02T04:57:02.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:01 smithi086 ceph-mon[24811]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:57:02.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:01 smithi086 ceph-mon[29505]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:57:02.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:01 smithi192 ceph-mon[25697]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:57:03.834 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=error ts=2024-04-02T04:57:03.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.192:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.192 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.86:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.86 because it doesn't contain any IP SANs" 2024-04-02T04:57:03.834 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:57:03.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.192:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.192 because it doesn't contain any IP SANs" 2024-04-02T04:57:03.834 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:57:03.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.86:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.86 because it doesn't contain any IP SANs" 2024-04-02T04:57:04.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:03 smithi192 ceph-mon[25697]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:57:04.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:03 smithi086 ceph-mon[24811]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:57:04.375 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:03 smithi086 ceph-mon[29505]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:57:05.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:05 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:57:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:57:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:05 smithi192 ceph-mon[25697]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:57:06.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:05 smithi086 ceph-mon[24811]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:57:06.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:05 smithi086 ceph-mon[29505]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:57:08.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:07 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:57:08.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:07 smithi192 ceph-mon[25697]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:57:08.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:07 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:57:08.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:07 smithi086 ceph-mon[29505]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:57:08.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:07 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:57:08.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:07 smithi086 ceph-mon[24811]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:57:10.130 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:09 smithi086 ceph-mon[24811]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:57:10.130 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:09 smithi086 ceph-mon[29505]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:57:10.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:09 smithi192 ceph-mon[25697]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:57:10.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:10 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:57:10] "GET /metrics HTTP/1.1" 200 208013 "" "Prometheus/2.33.4" 2024-04-02T04:57:12.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:11 smithi086 ceph-mon[24811]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:57:12.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:11 smithi086 ceph-mon[29505]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:57:12.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:11 smithi192 ceph-mon[25697]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:57:13.834 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:13 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=error ts=2024-04-02T04:57:13.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.86:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.86 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.192:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.192 because it doesn't contain any IP SANs" 2024-04-02T04:57:13.834 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:13 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:57:13.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.86:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.86 because it doesn't contain any IP SANs" 2024-04-02T04:57:13.834 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:13 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:57:13.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.192:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.192 because it doesn't contain any IP SANs" 2024-04-02T04:57:14.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:13 smithi192 ceph-mon[25697]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:57:14.247 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:13 smithi086 ceph-mon[24811]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:57:14.247 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:13 smithi086 ceph-mon[29505]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:57:15.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:15 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:57:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:57:16.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:15 smithi192 ceph-mon[25697]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:57:16.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:15 smithi086 ceph-mon[24811]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:57:16.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:15 smithi086 ceph-mon[29505]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:57:18.073 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:17 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:57:18.073 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:17 smithi086 ceph-mon[29505]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:57:18.073 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:17 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:57:18.073 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:17 smithi086 ceph-mon[24811]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:57:18.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:17 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:57:18.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:17 smithi192 ceph-mon[25697]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:57:18.478 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch ps' 2024-04-02T04:57:20.022 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:19 smithi086 ceph-mon[29505]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:57:20.022 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:19 smithi086 ceph-mon[24811]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:57:20.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:19 smithi192 ceph-mon[25697]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:57:20.334 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:57:20] "GET /metrics HTTP/1.1" 200 208005 "" "Prometheus/2.33.4" 2024-04-02T04:57:20.404 INFO:teuthology.orchestra.run.smithi086.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T04:57:20.405 INFO:teuthology.orchestra.run.smithi086.stdout:alertmanager.a smithi086 *:9093,9094 running (6m) 2m ago 7m 25.9M - ba2b418f427c 24a36455c114 2024-04-02T04:57:20.405 INFO:teuthology.orchestra.run.smithi086.stdout:grafana.a smithi192 *:3000 running (7m) 83s ago 7m 53.5M - 8.3.5 dad864ee21e9 6517aa8b3121 2024-04-02T04:57:20.405 INFO:teuthology.orchestra.run.smithi086.stdout:iscsi.foo.smithi086.rsbqwx smithi086 running (6m) 2m ago 6m 50.7M - 3.5 e1d6a67b021e 828835e36fd4 2024-04-02T04:57:20.405 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.x smithi192 *:8443,9283 running (2m) 83s ago 11m 449M - 19.0.0-1979-g6cec793d 78f830190839 fd618149d6b4 2024-04-02T04:57:20.405 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.y smithi086 *:9283 running (13m) 2m ago 13m 461M - 17.2.0 e1d6a67b021e 02f336a19fd6 2024-04-02T04:57:20.405 INFO:teuthology.orchestra.run.smithi086.stdout:mon.a smithi086 running (13m) 2m ago 13m 49.9M 2048M 17.2.0 e1d6a67b021e b6c507ab96f8 2024-04-02T04:57:20.405 INFO:teuthology.orchestra.run.smithi086.stdout:mon.b smithi192 running (11m) 83s ago 11m 42.0M 2048M 17.2.0 e1d6a67b021e 4dadd8d1c318 2024-04-02T04:57:20.405 INFO:teuthology.orchestra.run.smithi086.stdout:mon.c smithi086 running (12m) 2m ago 12m 36.6M 2048M 17.2.0 e1d6a67b021e 49275483a2d4 2024-04-02T04:57:20.405 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.a smithi086 *:9100 running (8m) 2m ago 8m 17.4M - 1dbe0e931976 0151f4e72bd9 2024-04-02T04:57:20.406 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.b smithi192 *:9100 running (7m) 83s ago 7m 17.0M - 1dbe0e931976 27476f65f84f 2024-04-02T04:57:20.406 INFO:teuthology.orchestra.run.smithi086.stdout:osd.0 smithi086 running (11m) 2m ago 11m 52.7M 4016M 17.2.0 e1d6a67b021e 6d90c60d1c71 2024-04-02T04:57:20.406 INFO:teuthology.orchestra.run.smithi086.stdout:osd.1 smithi086 running (11m) 2m ago 11m 51.5M 4016M 17.2.0 e1d6a67b021e 203689f71603 2024-04-02T04:57:20.406 INFO:teuthology.orchestra.run.smithi086.stdout:osd.2 smithi086 running (10m) 2m ago 10m 46.6M 4016M 17.2.0 e1d6a67b021e 472c860a9af8 2024-04-02T04:57:20.406 INFO:teuthology.orchestra.run.smithi086.stdout:osd.3 smithi086 running (10m) 2m ago 10m 48.5M 4016M 17.2.0 e1d6a67b021e 327538dac44e 2024-04-02T04:57:20.406 INFO:teuthology.orchestra.run.smithi086.stdout:osd.4 smithi192 running (9m) 83s ago 9m 50.2M 4272M 17.2.0 e1d6a67b021e 6535d7919a2d 2024-04-02T04:57:20.406 INFO:teuthology.orchestra.run.smithi086.stdout:osd.5 smithi192 running (9m) 83s ago 9m 50.7M 4272M 17.2.0 e1d6a67b021e 7259249605f2 2024-04-02T04:57:20.406 INFO:teuthology.orchestra.run.smithi086.stdout:osd.6 smithi192 running (9m) 83s ago 9m 47.5M 4272M 17.2.0 e1d6a67b021e bb7d171b02a1 2024-04-02T04:57:20.406 INFO:teuthology.orchestra.run.smithi086.stdout:osd.7 smithi192 running (8m) 83s ago 8m 50.0M 4272M 17.2.0 e1d6a67b021e 95f5a0353999 2024-04-02T04:57:20.406 INFO:teuthology.orchestra.run.smithi086.stdout:prometheus.a smithi192 *:9095 running (6m) 83s ago 8m 67.6M - 514e6a882f6e 782ae1d5e7f9 2024-04-02T04:57:20.406 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi086.bxzxlp smithi086 *:8000 running (6m) 2m ago 6m 72.7M - 17.2.0 e1d6a67b021e 6725785fbc53 2024-04-02T04:57:20.407 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi192.pmlbot smithi192 *:8000 running (6m) 83s ago 6m 72.1M - 17.2.0 e1d6a67b021e 5ae3638f8b1c 2024-04-02T04:57:21.018 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph versions' 2024-04-02T04:57:22.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:21 smithi086 ceph-mon[24811]: from='client.14994 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:57:22.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:21 smithi086 ceph-mon[24811]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:57:22.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:21 smithi086 ceph-mon[29505]: from='client.14994 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:57:22.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:21 smithi086 ceph-mon[29505]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:57:22.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:21 smithi192 ceph-mon[25697]: from='client.14994 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:57:22.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:21 smithi192 ceph-mon[25697]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:57:22.822 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T04:57:22.822 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": { 2024-04-02T04:57:22.822 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-02T04:57:22.822 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T04:57:22.822 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": { 2024-04-02T04:57:22.822 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-02T04:57:22.822 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 1 2024-04-02T04:57:22.822 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T04:57:22.822 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": { 2024-04-02T04:57:22.822 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-02T04:57:22.823 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T04:57:22.823 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": {}, 2024-04-02T04:57:22.823 INFO:teuthology.orchestra.run.smithi086.stdout: "rgw": { 2024-04-02T04:57:22.823 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-02T04:57:22.823 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T04:57:22.823 INFO:teuthology.orchestra.run.smithi086.stdout: "overall": { 2024-04-02T04:57:22.823 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14, 2024-04-02T04:57:22.823 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 1 2024-04-02T04:57:22.823 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-04-02T04:57:22.823 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T04:57:23.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:22 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/3316439991' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T04:57:23.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:22 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/3316439991' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T04:57:23.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:22 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/3316439991' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T04:57:23.463 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph -s' 2024-04-02T04:57:23.767 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=error ts=2024-04-02T04:57:23.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.86:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.86 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.192:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.192 because it doesn't contain any IP SANs" 2024-04-02T04:57:23.767 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:57:23.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.192:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.192 because it doesn't contain any IP SANs" 2024-04-02T04:57:23.767 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:57:23.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.86:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.86 because it doesn't contain any IP SANs" 2024-04-02T04:57:24.069 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:23 smithi086 ceph-mon[29505]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:57:24.069 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:23 smithi086 ceph-mon[24811]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:57:24.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:23 smithi192 ceph-mon[25697]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:57:25.310 INFO:teuthology.orchestra.run.smithi086.stdout: cluster: 2024-04-02T04:57:25.311 INFO:teuthology.orchestra.run.smithi086.stdout: id: 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 2024-04-02T04:57:25.311 INFO:teuthology.orchestra.run.smithi086.stdout: health: HEALTH_OK 2024-04-02T04:57:25.311 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-02T04:57:25.311 INFO:teuthology.orchestra.run.smithi086.stdout: services: 2024-04-02T04:57:25.311 INFO:teuthology.orchestra.run.smithi086.stdout: mon: 3 daemons, quorum a,c,b (age 11m) 2024-04-02T04:57:25.311 INFO:teuthology.orchestra.run.smithi086.stdout: mgr: y(active, since 8m), standbys: x 2024-04-02T04:57:25.311 INFO:teuthology.orchestra.run.smithi086.stdout: osd: 8 osds: 8 up (since 8m), 8 in (since 9m) 2024-04-02T04:57:25.311 INFO:teuthology.orchestra.run.smithi086.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-04-02T04:57:25.311 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-02T04:57:25.311 INFO:teuthology.orchestra.run.smithi086.stdout: data: 2024-04-02T04:57:25.311 INFO:teuthology.orchestra.run.smithi086.stdout: pools: 6 pools, 161 pgs 2024-04-02T04:57:25.311 INFO:teuthology.orchestra.run.smithi086.stdout: objects: 241 objects, 585 KiB 2024-04-02T04:57:25.311 INFO:teuthology.orchestra.run.smithi086.stdout: usage: 80 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:57:25.311 INFO:teuthology.orchestra.run.smithi086.stdout: pgs: 161 active+clean 2024-04-02T04:57:25.311 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-02T04:57:25.312 INFO:teuthology.orchestra.run.smithi086.stdout: io: 2024-04-02T04:57:25.312 INFO:teuthology.orchestra.run.smithi086.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-04-02T04:57:25.312 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-02T04:57:25.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:25 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:57:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:57:25.935 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph health detail' 2024-04-02T04:57:26.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:25 smithi192 ceph-mon[25697]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:57:26.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:25 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/404627681' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-02T04:57:26.228 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:25 smithi086 ceph-mon[29505]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:57:26.228 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:25 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/404627681' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-02T04:57:26.228 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:25 smithi086 ceph-mon[24811]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:57:26.228 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:25 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/404627681' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-02T04:57:27.835 INFO:teuthology.orchestra.run.smithi086.stdout:HEALTH_OK 2024-04-02T04:57:28.139 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:27 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:57:28.139 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:27 smithi086 ceph-mon[24811]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:57:28.139 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:27 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/2468360816' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T04:57:28.139 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:27 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:57:28.140 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:27 smithi086 ceph-mon[29505]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:57:28.140 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:27 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/2468360816' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T04:57:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:27 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:57:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:27 smithi192 ceph-mon[25697]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:57:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:27 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/2468360816' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T04:57:28.395 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 2'"'"'' 2024-04-02T04:57:30.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:29 smithi086 ceph-mon[24811]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:57:30.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:29 smithi086 ceph-mon[29505]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:57:30.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:29 smithi192 ceph-mon[25697]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:57:30.214 INFO:teuthology.orchestra.run.smithi086.stdout:true 2024-04-02T04:57:30.584 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:57:30] "GET /metrics HTTP/1.1" 200 208015 "" "Prometheus/2.33.4" 2024-04-02T04:57:30.752 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph mgr fail' 2024-04-02T04:57:31.059 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:30 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1763765051' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T04:57:31.060 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:30 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1763765051' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T04:57:31.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:30 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1763765051' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T04:57:32.052 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:31 smithi086 ceph-mon[24811]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:57:32.053 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:31 smithi086 ceph-mon[29505]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:57:32.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:31 smithi192 ceph-mon[25697]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:57:33.142 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:32 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/828350302' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-02T04:57:33.142 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:32 smithi086 ceph-mon[29505]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-02T04:57:33.142 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:32 smithi086 ceph-mon[29505]: osdmap e94: 8 total, 8 up, 8 in 2024-04-02T04:57:33.143 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:32 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/828350302' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-02T04:57:33.143 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:32 smithi086 ceph-mon[24811]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-02T04:57:33.143 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:32 smithi086 ceph-mon[24811]: osdmap e94: 8 total, 8 up, 8 in 2024-04-02T04:57:33.143 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:32 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:57:32.893+0000 7f4097e68700 -1 mgr handle_mgr_map I was active but no longer am 2024-04-02T04:57:33.143 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:32 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ignoring --setuser ceph since I am not root 2024-04-02T04:57:33.143 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:32 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ignoring --setgroup ceph since I am not root 2024-04-02T04:57:33.143 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:32 smithi086 ceph-mgr[25044]: -- 172.21.15.86:0/2300422370 <== mon.1 v2:172.21.15.86:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5623f621e340 con 0x5623f61f6c00 2024-04-02T04:57:33.144 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:32 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:57:32.952Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=7 err="Post \"https://172.21.15.192:8443/api/prometheus_receiver\": dial tcp 172.21.15.192:8443: connect: connection refused" 2024-04-02T04:57:33.145 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:32 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/828350302' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-02T04:57:33.145 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:32 smithi192 ceph-mon[25697]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-02T04:57:33.145 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:32 smithi192 ceph-mon[25697]: osdmap e94: 8 total, 8 up, 8 in 2024-04-02T04:57:33.145 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:04:57:32] ENGINE Bus STOPPING 2024-04-02T04:57:33.145 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:04:57:32] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-02T04:57:33.145 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:04:57:32] ENGINE Bus STOPPED 2024-04-02T04:57:33.145 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:04:57:32] ENGINE Bus STARTING 2024-04-02T04:57:33.145 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:33 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:04:57:33] ENGINE Serving on http://:::9283 2024-04-02T04:57:33.145 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:33 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:04:57:33] ENGINE Bus STARTED 2024-04-02T04:57:33.563 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'sleep 180' 2024-04-02T04:57:33.834 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:57:33.414+0000 7f71f0ce3000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-02T04:57:33.834 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:57:33.509+0000 7f71f0ce3000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-02T04:57:33.834 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=error ts=2024-04-02T04:57:33.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.192:8443/api/prometheus_receiver\": dial tcp 172.21.15.192:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.86:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.86 because it doesn't contain any IP SANs" 2024-04-02T04:57:33.834 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:57:33.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.86:8443//api/prometheus_receiver\": dial tcp 172.21.15.86:8443: connect: connection refused" 2024-04-02T04:57:33.834 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:57:33.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.192:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.192 because it doesn't contain any IP SANs" 2024-04-02T04:57:34.152 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[24811]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-02T04:57:34.152 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[24811]: mgrmap e22: x(active, starting, since 0.282615s) 2024-04-02T04:57:34.153 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T04:57:34.153 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T04:57:34.153 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T04:57:34.153 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T04:57:34.153 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T04:57:34.153 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T04:57:34.153 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T04:57:34.153 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T04:57:34.153 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T04:57:34.153 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T04:57:34.153 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T04:57:34.153 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T04:57:34.153 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T04:57:34.154 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T04:57:34.154 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T04:57:34.154 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[24811]: Manager daemon x is now available 2024-04-02T04:57:34.154 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:34.154 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[24811]: Queued rgw.foo for migration 2024-04-02T04:57:34.154 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[24811]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-04-02T04:57:34.154 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:34.154 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:57:34.154 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:57:34.154 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:57:34.154 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-02T04:57:34.154 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-02T04:57:34.154 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:34.155 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:34.155 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[29505]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-02T04:57:34.155 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[29505]: mgrmap e22: x(active, starting, since 0.282615s) 2024-04-02T04:57:34.155 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T04:57:34.155 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T04:57:34.155 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T04:57:34.155 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T04:57:34.155 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T04:57:34.155 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T04:57:34.155 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T04:57:34.155 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T04:57:34.155 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T04:57:34.155 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T04:57:34.156 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T04:57:34.156 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T04:57:34.156 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T04:57:34.156 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T04:57:34.156 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T04:57:34.156 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[29505]: Manager daemon x is now available 2024-04-02T04:57:34.156 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:34.156 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[29505]: Queued rgw.foo for migration 2024-04-02T04:57:34.156 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[29505]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-04-02T04:57:34.156 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:34.156 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:57:34.156 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:57:34.156 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:57:34.156 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-02T04:57:34.156 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-02T04:57:34.157 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:34.157 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:33 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:34.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:33 smithi192 ceph-mon[25697]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-02T04:57:34.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:33 smithi192 ceph-mon[25697]: mgrmap e22: x(active, starting, since 0.282615s) 2024-04-02T04:57:34.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:33 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T04:57:34.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:33 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T04:57:34.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:33 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T04:57:34.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:33 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T04:57:34.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:33 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T04:57:34.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:33 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T04:57:34.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:33 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T04:57:34.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:33 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T04:57:34.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:33 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T04:57:34.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:33 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T04:57:34.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:33 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T04:57:34.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:33 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T04:57:34.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:33 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T04:57:34.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:33 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T04:57:34.180 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:33 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T04:57:34.180 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:33 smithi192 ceph-mon[25697]: Manager daemon x is now available 2024-04-02T04:57:34.180 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:33 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:34.180 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:33 smithi192 ceph-mon[25697]: Queued rgw.foo for migration 2024-04-02T04:57:34.180 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:33 smithi192 ceph-mon[25697]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-04-02T04:57:34.180 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:33 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:34.180 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:33 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:57:34.180 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:33 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:57:34.180 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:33 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:57:34.180 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:33 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-02T04:57:34.180 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:33 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-02T04:57:34.180 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:33 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:34.180 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:33 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:34.476 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:34 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:57:34.147+0000 7f71f0ce3000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-02T04:57:34.476 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:34 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:57:34.238+0000 7f71f0ce3000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-02T04:57:34.476 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:34 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:57:34.322+0000 7f71f0ce3000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-02T04:57:34.476 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:34 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:57:34.471+0000 7f71f0ce3000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-02T04:57:34.754 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:34 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:57:34.554+0000 7f71f0ce3000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-02T04:57:34.754 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:34 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:57:34.664+0000 7f71f0ce3000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-02T04:57:34.754 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:34 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:57:34.749+0000 7f71f0ce3000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-02T04:57:35.016 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:34 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:57:34.917+0000 7f71f0ce3000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-02T04:57:35.096 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:35 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:57:35.011+0000 7f71f0ce3000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-02T04:57:35.097 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:34 smithi086 ceph-mon[29505]: Deploying cephadm binary to smithi086 2024-04-02T04:57:35.097 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:34 smithi086 ceph-mon[29505]: Deploying cephadm binary to smithi192 2024-04-02T04:57:35.097 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:34 smithi086 ceph-mon[29505]: mgrmap e23: x(active, since 1.28946s) 2024-04-02T04:57:35.097 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:34 smithi086 ceph-mon[29505]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:57:35.097 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:34 smithi086 ceph-mon[29505]: [02/Apr/2024:04:57:34] ENGINE Bus STARTING 2024-04-02T04:57:35.097 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:34 smithi086 ceph-mon[29505]: [02/Apr/2024:04:57:34] ENGINE Serving on http://172.21.15.192:8765 2024-04-02T04:57:35.097 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:34 smithi086 ceph-mon[24811]: Deploying cephadm binary to smithi086 2024-04-02T04:57:35.097 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:34 smithi086 ceph-mon[24811]: Deploying cephadm binary to smithi192 2024-04-02T04:57:35.097 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:34 smithi086 ceph-mon[24811]: mgrmap e23: x(active, since 1.28946s) 2024-04-02T04:57:35.097 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:34 smithi086 ceph-mon[24811]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:57:35.098 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:34 smithi086 ceph-mon[24811]: [02/Apr/2024:04:57:34] ENGINE Bus STARTING 2024-04-02T04:57:35.098 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:34 smithi086 ceph-mon[24811]: [02/Apr/2024:04:57:34] ENGINE Serving on http://172.21.15.192:8765 2024-04-02T04:57:35.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:34 smithi192 ceph-mon[25697]: Deploying cephadm binary to smithi086 2024-04-02T04:57:35.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:34 smithi192 ceph-mon[25697]: Deploying cephadm binary to smithi192 2024-04-02T04:57:35.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:34 smithi192 ceph-mon[25697]: mgrmap e23: x(active, since 1.28946s) 2024-04-02T04:57:35.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:34 smithi192 ceph-mon[25697]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:57:35.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:34 smithi192 ceph-mon[25697]: [02/Apr/2024:04:57:34] ENGINE Bus STARTING 2024-04-02T04:57:35.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:34 smithi192 ceph-mon[25697]: [02/Apr/2024:04:57:34] ENGINE Serving on http://172.21.15.192:8765 2024-04-02T04:57:35.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:35 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:57:35] "GET /metrics HTTP/1.1" 200 34896 "" "Prometheus/2.33.4" 2024-04-02T04:57:36.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:35 smithi192 ceph-mon[25697]: [02/Apr/2024:04:57:34] ENGINE Serving on https://172.21.15.192:7150 2024-04-02T04:57:36.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:35 smithi192 ceph-mon[25697]: [02/Apr/2024:04:57:34] ENGINE Bus STARTED 2024-04-02T04:57:36.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:35 smithi192 ceph-mon[25697]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:57:36.179 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:36 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:57:36.046+0000 7f71f0ce3000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-02T04:57:36.180 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:35 smithi086 ceph-mon[24811]: [02/Apr/2024:04:57:34] ENGINE Serving on https://172.21.15.192:7150 2024-04-02T04:57:36.180 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:35 smithi086 ceph-mon[24811]: [02/Apr/2024:04:57:34] ENGINE Bus STARTED 2024-04-02T04:57:36.180 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:35 smithi086 ceph-mon[24811]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:57:36.180 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:35 smithi086 ceph-mon[29505]: [02/Apr/2024:04:57:34] ENGINE Serving on https://172.21.15.192:7150 2024-04-02T04:57:36.180 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:35 smithi086 ceph-mon[29505]: [02/Apr/2024:04:57:34] ENGINE Bus STARTED 2024-04-02T04:57:36.181 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:35 smithi086 ceph-mon[29505]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:57:36.454 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:36 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:57:36.173+0000 7f71f0ce3000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-02T04:57:36.454 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:36 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:57:36.362+0000 7f71f0ce3000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-02T04:57:36.454 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:36 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:57:36.449+0000 7f71f0ce3000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-02T04:57:36.834 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:36 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:57:36.584+0000 7f71f0ce3000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-02T04:57:37.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:36 smithi192 ceph-mon[25697]: mgrmap e24: x(active, since 3s) 2024-04-02T04:57:37.224 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:36 smithi086 ceph-mon[29505]: mgrmap e24: x(active, since 3s) 2024-04-02T04:57:37.224 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:36 smithi086 ceph-mon[24811]: mgrmap e24: x(active, since 3s) 2024-04-02T04:57:37.224 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:36 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:57:36.904+0000 7f71f0ce3000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-02T04:57:37.225 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:37 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:57:37.133+0000 7f71f0ce3000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-02T04:57:37.584 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:37 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:57:37.219+0000 7f71f0ce3000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-02T04:57:37.584 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:37 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:57:37.390+0000 7f71f0ce3000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-02T04:57:38.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:37 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:57:38.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:37 smithi192 ceph-mon[25697]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:57:38.327 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:37 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:57:38.327 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:37 smithi086 ceph-mon[29505]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:57:38.328 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:37 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:57:38.328 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:37 smithi086 ceph-mon[24811]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:57:38.584 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:38 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:57:38.323+0000 7f71f0ce3000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-02T04:57:38.908 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:38 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:57:38.649+0000 7f71f0ce3000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-02T04:57:38.908 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:38 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:57:38.746+0000 7f71f0ce3000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-02T04:57:38.908 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:38 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:57:38.904+0000 7f71f0ce3000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-02T04:57:38.927 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:57:38 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[42606]: ts=2024-04-02T04:57:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: |\n OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked down and back up at {{ $value | humanize }} times once a minute for 5 minutes. This could indicate a network issue (latency, packet drop, disruption) on the clusters \"cluster network\". Check the network environment on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSD's to flap (mark each other out)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"172.21.15.86:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"172.21.15.192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-02T04:57:38.927 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:57:38 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[42606]: ts=2024-04-02T04:57:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: |\n OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by more than 30% from average PG count.\n summary: PG allocations are not balanced across devices\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"172.21.15.86:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"172.21.15.192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-02T04:57:39.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:38 smithi192 ceph-mon[25697]: Standby manager daemon y started 2024-04-02T04:57:39.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:38 smithi192 ceph-mon[25697]: from='mgr.? 172.21.15.86:0/465326047' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-02T04:57:39.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:38 smithi192 ceph-mon[25697]: from='mgr.? 172.21.15.86:0/465326047' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T04:57:39.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:38 smithi192 ceph-mon[25697]: from='mgr.? 172.21.15.86:0/465326047' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-02T04:57:39.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:38 smithi192 ceph-mon[25697]: from='mgr.? 172.21.15.86:0/465326047' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T04:57:39.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:38 smithi086 ceph-mon[24811]: Standby manager daemon y started 2024-04-02T04:57:39.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:38 smithi086 ceph-mon[24811]: from='mgr.? 172.21.15.86:0/465326047' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-02T04:57:39.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:38 smithi086 ceph-mon[24811]: from='mgr.? 172.21.15.86:0/465326047' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T04:57:39.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:38 smithi086 ceph-mon[24811]: from='mgr.? 172.21.15.86:0/465326047' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-02T04:57:39.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:38 smithi086 ceph-mon[24811]: from='mgr.? 172.21.15.86:0/465326047' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T04:57:39.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:38 smithi086 ceph-mon[29505]: Standby manager daemon y started 2024-04-02T04:57:39.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:38 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.86:0/465326047' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-02T04:57:39.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:38 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.86:0/465326047' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T04:57:39.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:38 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.86:0/465326047' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-02T04:57:39.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:38 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.86:0/465326047' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T04:57:39.335 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:38 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: [02/Apr/2024:04:57:38] ENGINE Bus STARTING 2024-04-02T04:57:39.335 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:39 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: CherryPy Checker: 2024-04-02T04:57:39.335 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:39 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: The Application mounted at '' has an empty config. 2024-04-02T04:57:39.335 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:39 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: 2024-04-02T04:57:39.335 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:39 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: [02/Apr/2024:04:57:39] ENGINE Serving on http://:::9283 2024-04-02T04:57:39.335 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:39 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: [02/Apr/2024:04:57:39] ENGINE Bus STARTED 2024-04-02T04:57:40.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:39 smithi086 ceph-mon[24811]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:57:40.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:39 smithi086 ceph-mon[24811]: mgrmap e25: x(active, since 6s), standbys: y 2024-04-02T04:57:40.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:39 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T04:57:40.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:39 smithi086 ceph-mon[29505]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:57:40.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:39 smithi086 ceph-mon[29505]: mgrmap e25: x(active, since 6s), standbys: y 2024-04-02T04:57:40.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:39 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T04:57:40.300 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:39 smithi192 ceph-mon[25697]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:57:40.300 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:39 smithi192 ceph-mon[25697]: mgrmap e25: x(active, since 6s), standbys: y 2024-04-02T04:57:40.300 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:39 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T04:57:40.384 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:40 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:57:40] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:57:41.279 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:40 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:57:40.982Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=6 err="Post \"https://172.21.15.86:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.86 because it doesn't contain any IP SANs" 2024-04-02T04:57:42.009 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:41 smithi192 ceph-mon[25697]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-02T04:57:42.010 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:41 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:42.010 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:41 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:42.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:41 smithi086 ceph-mon[29505]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-02T04:57:42.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:41 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:42.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:41 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:42.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:41 smithi086 ceph-mon[24811]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-02T04:57:42.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:41 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:42.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:41 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:43.584 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:43 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=error ts=2024-04-02T04:57:43.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.192:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.192 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.86:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.86 because it doesn't contain any IP SANs" 2024-04-02T04:57:43.584 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:43 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:57:43.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.86:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.86 because it doesn't contain any IP SANs" 2024-04-02T04:57:43.584 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:43 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=warn ts=2024-04-02T04:57:43.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.192:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.192 because it doesn't contain any IP SANs" 2024-04-02T04:57:43.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:43 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:43.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:43 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:43.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:43 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:43.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:43 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:43.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:43 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:43.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:43 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:43.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:43 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:43.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:43 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:43.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:43 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:43.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:43 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:43.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:43 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:43.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:43 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:43.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:43 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:43.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:43 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:43.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:43 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:43.586 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:43 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:43.586 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:43 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:43.586 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:43 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:43.586 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:43 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:43.586 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:43 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:43.586 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:43 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:43.586 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:43 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:43.586 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:43 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:43.586 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:43 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:43.586 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:43 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:43.586 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:43 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:43.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:43 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:43.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:43 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:43.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:43 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:43.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:43 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:43.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:43 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:43.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:43 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:43.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:43 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:43.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:43 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:43.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:43 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:43.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:43 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:43.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:43 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:43.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:43 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:43.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:43 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:44.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:44 smithi192 ceph-mon[25697]: Adjusting osd_memory_target on smithi086 to 2992M 2024-04-02T04:57:44.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:44 smithi192 ceph-mon[25697]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-02T04:57:44.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:44 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:44.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:44 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:44.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:44 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:44.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:44 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:44.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:44 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:44.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:44 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:44.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:44 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:44.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:44 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:44.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:44 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:44.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:44 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:44.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:44 smithi192 ceph-mon[25697]: Adjusting osd_memory_target on smithi192 to 3248M 2024-04-02T04:57:44.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:44 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:44.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:44 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:57:44.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:44 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:57:44.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:44 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:44.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:44 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:44.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:44 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:44.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:44 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:44.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:44 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:44.788 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[24811]: Adjusting osd_memory_target on smithi086 to 2992M 2024-04-02T04:57:44.789 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[24811]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-02T04:57:44.789 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:44.789 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:44.789 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:44.789 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:44.789 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:44.789 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:44.789 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:44.789 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:44.789 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:44.789 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:44.789 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[24811]: Adjusting osd_memory_target on smithi192 to 3248M 2024-04-02T04:57:44.790 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:44.790 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:57:44.790 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:57:44.790 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:44.790 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:44.790 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:44.790 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:44.790 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:44.790 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[29505]: Adjusting osd_memory_target on smithi086 to 2992M 2024-04-02T04:57:44.790 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[29505]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-02T04:57:44.790 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:44.791 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:44.791 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:44.791 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:44.791 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:44.791 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:44.791 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:44.791 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:44.791 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:44.791 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:57:44.791 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[29505]: Adjusting osd_memory_target on smithi192 to 3248M 2024-04-02T04:57:44.791 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:44.791 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:57:44.791 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:57:44.791 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:44.791 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:44.792 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:44.792 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:44.792 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:44 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:45.084 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:44 smithi086 systemd[1]: Stopping Ceph node-exporter.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T04:57:45.084 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:44 smithi086 bash[58507]: Error: no container with name or ID "ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter.a" found: no such container 2024-04-02T04:57:45.361 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:45 smithi086 podman[58521]: 2024-04-02 04:57:45.085142315 +0000 UTC m=+0.111465969 container died 0151f4e72bd9ab8de43694ed7be25b1a230c9320106f0c2b09ce93fde02a7d94 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-02T04:57:45.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:45 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:57:45] "GET /metrics HTTP/1.1" 200 34896 "" "Prometheus/2.33.4" 2024-04-02T04:57:45.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:45 smithi192 ceph-mon[25697]: Updating smithi086:/etc/ceph/ceph.conf 2024-04-02T04:57:45.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:45 smithi192 ceph-mon[25697]: Updating smithi192:/etc/ceph/ceph.conf 2024-04-02T04:57:45.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:45 smithi192 ceph-mon[25697]: Updating smithi192:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T04:57:45.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:45 smithi192 ceph-mon[25697]: Updating smithi086:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T04:57:45.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:45 smithi192 ceph-mon[25697]: Updating smithi192:/etc/ceph/ceph.client.admin.keyring 2024-04-02T04:57:45.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:45 smithi192 ceph-mon[25697]: Updating smithi086:/etc/ceph/ceph.client.admin.keyring 2024-04-02T04:57:45.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:45 smithi192 ceph-mon[25697]: Updating smithi192:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-02T04:57:45.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:45 smithi192 ceph-mon[25697]: Updating smithi086:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-02T04:57:45.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:45 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:45.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:45 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:45.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:45 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:45.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:45 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:45.737 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:45 smithi086 ceph-mon[29505]: Updating smithi086:/etc/ceph/ceph.conf 2024-04-02T04:57:45.737 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:45 smithi086 ceph-mon[29505]: Updating smithi192:/etc/ceph/ceph.conf 2024-04-02T04:57:45.737 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:45 smithi086 ceph-mon[29505]: Updating smithi192:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T04:57:45.737 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:45 smithi086 ceph-mon[29505]: Updating smithi086:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T04:57:45.737 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:45 smithi086 ceph-mon[29505]: Updating smithi192:/etc/ceph/ceph.client.admin.keyring 2024-04-02T04:57:45.737 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:45 smithi086 ceph-mon[29505]: Updating smithi086:/etc/ceph/ceph.client.admin.keyring 2024-04-02T04:57:45.737 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:45 smithi086 ceph-mon[29505]: Updating smithi192:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-02T04:57:45.737 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:45 smithi086 ceph-mon[29505]: Updating smithi086:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-02T04:57:45.738 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:45 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:45.738 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:45 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:45.738 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:45 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:45.738 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:45 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:45.738 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:45 smithi086 ceph-mon[24811]: Updating smithi086:/etc/ceph/ceph.conf 2024-04-02T04:57:45.738 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:45 smithi086 ceph-mon[24811]: Updating smithi192:/etc/ceph/ceph.conf 2024-04-02T04:57:45.738 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:45 smithi086 ceph-mon[24811]: Updating smithi192:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T04:57:45.738 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:45 smithi086 ceph-mon[24811]: Updating smithi086:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T04:57:45.738 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:45 smithi086 ceph-mon[24811]: Updating smithi192:/etc/ceph/ceph.client.admin.keyring 2024-04-02T04:57:45.739 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:45 smithi086 ceph-mon[24811]: Updating smithi086:/etc/ceph/ceph.client.admin.keyring 2024-04-02T04:57:45.739 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:45 smithi086 ceph-mon[24811]: Updating smithi192:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-02T04:57:45.739 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:45 smithi086 ceph-mon[24811]: Updating smithi086:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-02T04:57:45.739 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:45 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:45.739 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:45 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:45.739 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:45 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:45.739 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:45 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:45.739 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:45 smithi086 podman[58521]: 2024-04-02 04:57:45.457674835 +0000 UTC m=+0.483998478 container cleanup 0151f4e72bd9ab8de43694ed7be25b1a230c9320106f0c2b09ce93fde02a7d94 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-02T04:57:45.739 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:45 smithi086 bash[58521]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a 2024-04-02T04:57:45.739 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:45 smithi086 bash[58551]: Error: no container with name or ID "ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter.a" found: no such container 2024-04-02T04:57:46.066 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:45 smithi086 podman[58535]: 2024-04-02 04:57:45.733824332 +0000 UTC m=+0.648088482 container remove 0151f4e72bd9ab8de43694ed7be25b1a230c9320106f0c2b09ce93fde02a7d94 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-02T04:57:46.067 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:45 smithi086 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-02T04:57:46.067 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:45 smithi086 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-02T04:57:46.067 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:45 smithi086 systemd[1]: Stopped Ceph node-exporter.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T04:57:46.067 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:45 smithi086 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@node-exporter.a.service: Consumed 3.717s CPU time. 2024-04-02T04:57:46.067 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:45 smithi086 systemd[1]: Starting Ceph node-exporter.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T04:57:46.067 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:45 smithi086 podman[58621]: 2024-04-02 04:57:45.951391999 +0000 UTC m=+0.106974597 container create 843a5e9887fba59327154514ed1f112b3c2406c7bf7efe073a54954d144d7f2e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-02T04:57:46.067 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:45 smithi086 podman[58621]: 2024-04-02 04:57:45.86191437 +0000 UTC m=+0.017496925 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-04-02T04:57:46.425 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-mon[24811]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-02T04:57:46.426 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-mon[24811]: Reconfiguring daemon node-exporter.a on smithi086 2024-04-02T04:57:46.426 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-mon[24811]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-02T04:57:46.426 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:46.426 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:46.426 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-mon[29505]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-02T04:57:46.426 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-mon[29505]: Reconfiguring daemon node-exporter.a on smithi086 2024-04-02T04:57:46.426 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-mon[29505]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-02T04:57:46.426 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:46.426 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:46.427 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 podman[58621]: 2024-04-02 04:57:46.137149246 +0000 UTC m=+0.292731798 container init 843a5e9887fba59327154514ed1f112b3c2406c7bf7efe073a54954d144d7f2e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-02T04:57:46.427 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 podman[58621]: 2024-04-02 04:57:46.139918267 +0000 UTC m=+0.295500829 container start 843a5e9887fba59327154514ed1f112b3c2406c7bf7efe073a54954d144d7f2e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-02T04:57:46.427 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.143Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-04-02T04:57:46.427 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.143Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-04-02T04:57:46.427 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-04-02T04:57:46.427 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-02T04:57:46.427 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-04-02T04:57:46.427 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=arp 2024-04-02T04:57:46.428 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=bcache 2024-04-02T04:57:46.428 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=bonding 2024-04-02T04:57:46.428 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=btrfs 2024-04-02T04:57:46.428 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=conntrack 2024-04-02T04:57:46.428 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=cpu 2024-04-02T04:57:46.428 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-04-02T04:57:46.428 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=diskstats 2024-04-02T04:57:46.428 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=dmi 2024-04-02T04:57:46.428 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=edac 2024-04-02T04:57:46.428 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=entropy 2024-04-02T04:57:46.428 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-04-02T04:57:46.428 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=filefd 2024-04-02T04:57:46.428 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=filesystem 2024-04-02T04:57:46.428 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=hwmon 2024-04-02T04:57:46.429 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=infiniband 2024-04-02T04:57:46.429 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=ipvs 2024-04-02T04:57:46.429 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=loadavg 2024-04-02T04:57:46.429 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=mdadm 2024-04-02T04:57:46.429 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=meminfo 2024-04-02T04:57:46.429 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=netclass 2024-04-02T04:57:46.429 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=netdev 2024-04-02T04:57:46.429 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=netstat 2024-04-02T04:57:46.429 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=nfs 2024-04-02T04:57:46.429 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=nfsd 2024-04-02T04:57:46.429 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=nvme 2024-04-02T04:57:46.429 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=os 2024-04-02T04:57:46.429 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-04-02T04:57:46.429 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=pressure 2024-04-02T04:57:46.430 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=rapl 2024-04-02T04:57:46.430 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=schedstat 2024-04-02T04:57:46.430 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=sockstat 2024-04-02T04:57:46.430 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=softnet 2024-04-02T04:57:46.430 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=stat 2024-04-02T04:57:46.430 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=tapestats 2024-04-02T04:57:46.430 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=textfile 2024-04-02T04:57:46.430 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-04-02T04:57:46.430 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=time 2024-04-02T04:57:46.430 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-04-02T04:57:46.430 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=uname 2024-04-02T04:57:46.430 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=vmstat 2024-04-02T04:57:46.431 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=xfs 2024-04-02T04:57:46.431 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:115 level=info collector=zfs 2024-04-02T04:57:46.431 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.144Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-04-02T04:57:46.431 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[58636]: ts=2024-04-02T04:57:46.145Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-04-02T04:57:46.431 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 bash[58621]: 843a5e9887fba59327154514ed1f112b3c2406c7bf7efe073a54954d144d7f2e 2024-04-02T04:57:46.431 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 04:57:46 smithi086 systemd[1]: Started Ceph node-exporter.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T04:57:46.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:46 smithi192 ceph-mon[25697]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-02T04:57:46.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:46 smithi192 ceph-mon[25697]: Reconfiguring daemon node-exporter.a on smithi086 2024-04-02T04:57:46.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:46 smithi192 ceph-mon[25697]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-02T04:57:46.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:46 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:46.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:46 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:47.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:47 smithi192 ceph-mon[25697]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-02T04:57:47.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:47 smithi192 ceph-mon[25697]: Reconfiguring daemon alertmanager.a on smithi086 2024-04-02T04:57:47.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:47 smithi086 ceph-mon[24811]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-02T04:57:47.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:47 smithi086 ceph-mon[24811]: Reconfiguring daemon alertmanager.a on smithi086 2024-04-02T04:57:47.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:47 smithi086 ceph-mon[29505]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-02T04:57:47.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:47 smithi086 ceph-mon[29505]: Reconfiguring daemon alertmanager.a on smithi086 2024-04-02T04:57:48.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:48 smithi192 ceph-mon[25697]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:57:48.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:48 smithi192 ceph-mon[25697]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-02T04:57:48.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:48 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:57:48.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:48 smithi086 ceph-mon[24811]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:57:48.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:48 smithi086 ceph-mon[24811]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-02T04:57:48.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:48 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:57:48.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:48 smithi086 ceph-mon[29505]: from='client.14796 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:57:48.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:48 smithi086 ceph-mon[29505]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-02T04:57:48.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:48 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:57:50.384 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:57:50 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:57:50] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:57:50.658 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:50 smithi086 ceph-mon[24811]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-02T04:57:50.659 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:50 smithi086 ceph-mon[29505]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-02T04:57:50.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:50 smithi192 ceph-mon[25697]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-02T04:57:51.940 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:51 smithi086 ceph-mon[24811]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-02T04:57:51.940 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:51 smithi086 ceph-mon[29505]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-02T04:57:52.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:51 smithi192 ceph-mon[25697]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-02T04:57:52.678 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:57:52 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[42606]: ts=2024-04-02T04:57:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: |\n Pool '{{ $labels.name }}' will be full in less than 5 days assuming the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed it's capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"172.21.15.86:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.192:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-02T04:57:53.040 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:53 smithi086 systemd[1]: Stopping Ceph alertmanager.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T04:57:53.334 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:53 smithi086 bash[58887]: Error: no container with name or ID "ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager.a" found: no such container 2024-04-02T04:57:53.334 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:53 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[49535]: level=info ts=2024-04-02T04:57:53.137Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-04-02T04:57:53.334 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:53 smithi086 podman[58899]: 2024-04-02 04:57:53.158213447 +0000 UTC m=+0.112579269 container died 24a36455c11437e04d8501bf54dc2d6d40b59f667fdb776472db88cca9e88dbc (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T04:57:53.725 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:53 smithi086 podman[58899]: 2024-04-02 04:57:53.462660057 +0000 UTC m=+0.417025884 container cleanup 24a36455c11437e04d8501bf54dc2d6d40b59f667fdb776472db88cca9e88dbc (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T04:57:53.725 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:53 smithi086 bash[58899]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a 2024-04-02T04:57:53.725 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:53 smithi086 bash[58927]: Error: no container with name or ID "ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager.a" found: no such container 2024-04-02T04:57:54.074 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:53 smithi086 ceph-mon[24811]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:57:54.075 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:53 smithi086 ceph-mon[29505]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:57:54.075 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:53 smithi086 podman[58914]: 2024-04-02 04:57:53.722070232 +0000 UTC m=+0.559856778 container remove 24a36455c11437e04d8501bf54dc2d6d40b59f667fdb776472db88cca9e88dbc (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T04:57:54.075 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:53 smithi086 podman[58914]: 2024-04-02 04:57:53.831231306 +0000 UTC m=+0.669017851 volume remove 911b09756c47529e737fc0641d1767d1ad2898bcfddd204fd20400492fb9c722 2024-04-02T04:57:54.075 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:53 smithi086 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@alertmanager.a.service: Deactivated successfully. 2024-04-02T04:57:54.075 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:53 smithi086 systemd[1]: Stopped Ceph alertmanager.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T04:57:54.075 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:53 smithi086 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@alertmanager.a.service: Consumed 1.059s CPU time. 2024-04-02T04:57:54.075 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:53 smithi086 systemd[1]: Starting Ceph alertmanager.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T04:57:54.334 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:54 smithi086 podman[58995]: 2024-04-02 04:57:53.974696557 +0000 UTC m=+0.019475265 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-04-02T04:57:54.334 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:54 smithi086 podman[58995]: 2024-04-02 04:57:54.198461557 +0000 UTC m=+0.243240257 volume create eb75ff9da27b94fb3cdbf822094e5a0f0f27572972d9a273c6273ac00c3d617a 2024-04-02T04:57:54.334 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:54 smithi086 podman[58995]: 2024-04-02 04:57:54.282106617 +0000 UTC m=+0.326885316 container create b2769c8e287a443c0b55b1ad86e7a1f48760f11038e4d4dc2c76557f77a58b52 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T04:57:54.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:53 smithi192 ceph-mon[25697]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:57:54.834 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:54 smithi086 podman[58995]: 2024-04-02 04:57:54.534873757 +0000 UTC m=+0.579652459 container init b2769c8e287a443c0b55b1ad86e7a1f48760f11038e4d4dc2c76557f77a58b52 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T04:57:54.834 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:54 smithi086 podman[58995]: 2024-04-02 04:57:54.537399139 +0000 UTC m=+0.582177846 container start b2769c8e287a443c0b55b1ad86e7a1f48760f11038e4d4dc2c76557f77a58b52 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T04:57:54.834 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:54 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[59010]: level=info ts=2024-04-02T04:57:54.547Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-04-02T04:57:54.834 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:54 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[59010]: level=info ts=2024-04-02T04:57:54.547Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-04-02T04:57:54.834 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:54 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[59010]: level=info ts=2024-04-02T04:57:54.549Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.86 port=9094 2024-04-02T04:57:54.835 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:54 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[59010]: level=info ts=2024-04-02T04:57:54.550Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-02T04:57:54.835 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:54 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[59010]: level=info ts=2024-04-02T04:57:54.575Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-02T04:57:54.835 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:54 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[59010]: level=info ts=2024-04-02T04:57:54.575Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-02T04:57:54.835 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:54 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[59010]: level=info ts=2024-04-02T04:57:54.578Z caller=main.go:518 msg=Listening address=:9093 2024-04-02T04:57:54.835 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:54 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[59010]: level=info ts=2024-04-02T04:57:54.578Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-04-02T04:57:54.835 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:54 smithi086 bash[58995]: b2769c8e287a443c0b55b1ad86e7a1f48760f11038e4d4dc2c76557f77a58b52 2024-04-02T04:57:54.835 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:54 smithi086 systemd[1]: Started Ceph alertmanager.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T04:57:55.645 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:55 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:57:55] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.33.4" 2024-04-02T04:57:55.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:55 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:55.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:55 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:55.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:55 smithi086 ceph-mon[24811]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T04:57:55.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:55 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T04:57:55.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:55 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T04:57:55.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:55 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:57:55.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:55 smithi086 ceph-mon[24811]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T04:57:55.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:55 smithi086 ceph-mon[24811]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:57:55.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:55 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:55.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:55 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:55.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:55 smithi086 ceph-mon[29505]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T04:57:55.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:55 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T04:57:55.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:55 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T04:57:55.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:55 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:57:55.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:55 smithi086 ceph-mon[29505]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T04:57:55.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:55 smithi086 ceph-mon[29505]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:57:55.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:55 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:55.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:55 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:57:55.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:55 smithi192 ceph-mon[25697]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T04:57:55.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:55 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T04:57:55.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:55 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T04:57:55.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:55 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:57:55.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:55 smithi192 ceph-mon[25697]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T04:57:55.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:55 smithi192 ceph-mon[25697]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:57:56.770 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:57:56 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[59010]: level=info ts=2024-04-02T04:57:56.551Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.001014198s 2024-04-02T04:57:58.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-mon[25697]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:57:58.179 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T04:57:57.744+0000 7f9e1a22d700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T04:57:58.180 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Creating ceph-iscsi config... 2024-04-02T04:57:58.180 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T04:57:58.180 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T04:57:58.180 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T04:57:58.180 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:57:58.180 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:57:58.181 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Traceback (most recent call last): 2024-04-02T04:57:58.181 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T04:57:58.181 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: return _run_code(code, main_globals, None, 2024-04-02T04:57:58.181 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T04:57:58.181 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: exec(code, run_globals) 2024-04-02T04:57:58.181 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T04:57:58.181 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T04:57:58.182 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T04:57:58.182 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T04:57:58.182 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T04:57:58.182 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T04:57:58.182 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T04:57:58.182 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:57:58.182 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:57:58.182 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Traceback (most recent call last): 2024-04-02T04:57:58.183 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T04:57:58.183 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T04:57:58.183 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T04:57:58.183 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T04:57:58.183 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T04:57:58.183 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: return self.event_loop.get_result(coro, timeout) 2024-04-02T04:57:58.183 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T04:57:58.183 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: return future.result(timeout) 2024-04-02T04:57:58.183 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T04:57:58.184 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: return self.__get_result() 2024-04-02T04:57:58.184 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T04:57:58.184 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: raise self._exception 2024-04-02T04:57:58.184 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T04:57:58.184 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: config_blobs=daemon_spec.final_config, 2024-04-02T04:57:58.184 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T04:57:58.184 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T04:57:58.184 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T04:57:58.185 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Creating ceph-iscsi config... 2024-04-02T04:57:58.185 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T04:57:58.185 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T04:57:58.185 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T04:57:58.185 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:57:58.185 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:57:58.186 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Traceback (most recent call last): 2024-04-02T04:57:58.186 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T04:57:58.186 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: return _run_code(code, main_globals, None, 2024-04-02T04:57:58.186 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T04:57:58.186 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: exec(code, run_globals) 2024-04-02T04:57:58.186 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T04:57:58.187 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T04:57:58.187 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T04:57:58.187 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T04:57:58.187 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T04:57:58.187 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T04:57:58.187 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T04:57:58.187 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:57:58.188 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:57:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:57:58.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:57 smithi086 ceph-mon[29505]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:57:58.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:57 smithi086 ceph-mon[24811]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:57:59.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:58 smithi086 ceph-mon[29505]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T04:57:59.334 INFO:journalctl@ceph.mon.c.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T04:57:59.335 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T04:57:59.335 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T04:57:59.335 INFO:journalctl@ceph.mon.c.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T04:57:59.335 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:57:59.335 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:57:59.335 INFO:journalctl@ceph.mon.c.smithi086.stdout: Traceback (most recent call last): 2024-04-02T04:57:59.335 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T04:57:59.335 INFO:journalctl@ceph.mon.c.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T04:57:59.335 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T04:57:59.335 INFO:journalctl@ceph.mon.c.smithi086.stdout: exec(code, run_globals) 2024-04-02T04:57:59.335 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T04:57:59.335 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T04:57:59.335 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T04:57:59.335 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T04:57:59.336 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T04:57:59.336 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T04:57:59.336 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T04:57:59.336 INFO:journalctl@ceph.mon.c.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:57:59.336 INFO:journalctl@ceph.mon.c.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:57:59.336 INFO:journalctl@ceph.mon.c.smithi086.stdout: Traceback (most recent call last): 2024-04-02T04:57:59.336 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T04:57:59.336 INFO:journalctl@ceph.mon.c.smithi086.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T04:57:59.336 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T04:57:59.336 INFO:journalctl@ceph.mon.c.smithi086.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T04:57:59.336 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T04:57:59.336 INFO:journalctl@ceph.mon.c.smithi086.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-02T04:57:59.336 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T04:57:59.336 INFO:journalctl@ceph.mon.c.smithi086.stdout: return future.result(timeout) 2024-04-02T04:57:59.336 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T04:57:59.337 INFO:journalctl@ceph.mon.c.smithi086.stdout: return self.__get_result() 2024-04-02T04:57:59.337 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T04:57:59.337 INFO:journalctl@ceph.mon.c.smithi086.stdout: raise self._exception 2024-04-02T04:57:59.337 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T04:57:59.337 INFO:journalctl@ceph.mon.c.smithi086.stdout: config_blobs=daemon_spec.final_config, 2024-04-02T04:57:59.337 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T04:57:59.337 INFO:journalctl@ceph.mon.c.smithi086.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T04:57:59.337 INFO:journalctl@ceph.mon.c.smithi086.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T04:57:59.337 INFO:journalctl@ceph.mon.c.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T04:57:59.337 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T04:57:59.337 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T04:57:59.337 INFO:journalctl@ceph.mon.c.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T04:57:59.337 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:57:59.337 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:57:59.338 INFO:journalctl@ceph.mon.c.smithi086.stdout: Traceback (most recent call last): 2024-04-02T04:57:59.338 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T04:57:59.338 INFO:journalctl@ceph.mon.c.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T04:57:59.338 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T04:57:59.338 INFO:journalctl@ceph.mon.c.smithi086.stdout: exec(code, run_globals) 2024-04-02T04:57:59.338 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T04:57:59.338 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T04:57:59.338 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T04:57:59.338 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T04:57:59.338 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:58 smithi086 ceph-mon[24811]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T04:57:59.339 INFO:journalctl@ceph.mon.a.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T04:57:59.339 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T04:57:59.339 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T04:57:59.339 INFO:journalctl@ceph.mon.a.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T04:57:59.339 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:57:59.339 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:57:59.339 INFO:journalctl@ceph.mon.a.smithi086.stdout: Traceback (most recent call last): 2024-04-02T04:57:59.339 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T04:57:59.339 INFO:journalctl@ceph.mon.a.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T04:57:59.339 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T04:57:59.339 INFO:journalctl@ceph.mon.a.smithi086.stdout: exec(code, run_globals) 2024-04-02T04:57:59.339 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T04:57:59.339 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T04:57:59.339 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T04:57:59.340 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T04:57:59.340 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T04:57:59.340 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T04:57:59.340 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T04:57:59.340 INFO:journalctl@ceph.mon.a.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:57:59.340 INFO:journalctl@ceph.mon.a.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:57:59.340 INFO:journalctl@ceph.mon.a.smithi086.stdout: Traceback (most recent call last): 2024-04-02T04:57:59.340 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T04:57:59.340 INFO:journalctl@ceph.mon.a.smithi086.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T04:57:59.340 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T04:57:59.340 INFO:journalctl@ceph.mon.a.smithi086.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T04:57:59.340 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T04:57:59.340 INFO:journalctl@ceph.mon.a.smithi086.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-02T04:57:59.340 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T04:57:59.340 INFO:journalctl@ceph.mon.a.smithi086.stdout: return future.result(timeout) 2024-04-02T04:57:59.341 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T04:57:59.341 INFO:journalctl@ceph.mon.a.smithi086.stdout: return self.__get_result() 2024-04-02T04:57:59.341 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T04:57:59.341 INFO:journalctl@ceph.mon.a.smithi086.stdout: raise self._exception 2024-04-02T04:57:59.341 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T04:57:59.341 INFO:journalctl@ceph.mon.a.smithi086.stdout: config_blobs=daemon_spec.final_config, 2024-04-02T04:57:59.341 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T04:57:59.341 INFO:journalctl@ceph.mon.a.smithi086.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T04:57:59.341 INFO:journalctl@ceph.mon.a.smithi086.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T04:57:59.341 INFO:journalctl@ceph.mon.a.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T04:57:59.341 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T04:57:59.341 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T04:57:59.341 INFO:journalctl@ceph.mon.a.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T04:57:59.341 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:57:59.341 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:57:59.342 INFO:journalctl@ceph.mon.a.smithi086.stdout: Traceback (most recent call last): 2024-04-02T04:57:59.342 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T04:57:59.342 INFO:journalctl@ceph.mon.a.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T04:57:59.342 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T04:57:59.342 INFO:journalctl@ceph.mon.a.smithi086.stdout: exec(code, run_globals) 2024-04-02T04:57:59.342 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T04:57:59.342 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T04:57:59.342 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T04:57:59.342 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T04:57:59.342 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T04:57:59.342 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T04:57:59.342 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T04:57:59.342 INFO:journalctl@ceph.mon.a.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:57:59.343 INFO:journalctl@ceph.mon.a.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:57:59.343 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:58 smithi086 ceph-mon[24811]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-02T04:57:59.343 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:58 smithi086 ceph-mon[24811]: Reconfiguring daemon prometheus.a on smithi192 2024-04-02T04:57:59.343 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T04:57:59.343 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T04:57:59.343 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T04:57:59.343 INFO:journalctl@ceph.mon.c.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:57:59.343 INFO:journalctl@ceph.mon.c.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:57:59.343 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:58 smithi086 ceph-mon[29505]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-02T04:57:59.343 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:58 smithi086 ceph-mon[29505]: Reconfiguring daemon prometheus.a on smithi192 2024-04-02T04:57:59.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:58 smithi192 ceph-mon[25697]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T04:57:59.428 INFO:journalctl@ceph.mon.b.smithi192.stdout: Creating ceph-iscsi config... 2024-04-02T04:57:59.428 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T04:57:59.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T04:57:59.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T04:57:59.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:57:59.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:57:59.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: Traceback (most recent call last): 2024-04-02T04:57:59.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T04:57:59.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: return _run_code(code, main_globals, None, 2024-04-02T04:57:59.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T04:57:59.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: exec(code, run_globals) 2024-04-02T04:57:59.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T04:57:59.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T04:57:59.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T04:57:59.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T04:57:59.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T04:57:59.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T04:57:59.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T04:57:59.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:57:59.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:57:59.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: Traceback (most recent call last): 2024-04-02T04:57:59.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T04:57:59.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T04:57:59.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T04:57:59.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T04:57:59.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T04:57:59.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-02T04:57:59.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T04:57:59.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: return future.result(timeout) 2024-04-02T04:57:59.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T04:57:59.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: return self.__get_result() 2024-04-02T04:57:59.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T04:57:59.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: raise self._exception 2024-04-02T04:57:59.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T04:57:59.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: config_blobs=daemon_spec.final_config, 2024-04-02T04:57:59.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T04:57:59.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T04:57:59.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T04:57:59.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: Creating ceph-iscsi config... 2024-04-02T04:57:59.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T04:57:59.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T04:57:59.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T04:57:59.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:57:59.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:57:59.432 INFO:journalctl@ceph.mon.b.smithi192.stdout: Traceback (most recent call last): 2024-04-02T04:57:59.432 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T04:57:59.432 INFO:journalctl@ceph.mon.b.smithi192.stdout: return _run_code(code, main_globals, None, 2024-04-02T04:57:59.432 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T04:57:59.432 INFO:journalctl@ceph.mon.b.smithi192.stdout: exec(code, run_globals) 2024-04-02T04:57:59.432 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T04:57:59.432 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T04:57:59.432 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T04:57:59.432 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T04:57:59.432 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T04:57:59.432 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T04:57:59.432 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T04:57:59.432 INFO:journalctl@ceph.mon.b.smithi192.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:57:59.432 INFO:journalctl@ceph.mon.b.smithi192.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:57:59.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:58 smithi192 ceph-mon[25697]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-02T04:57:59.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:58 smithi192 ceph-mon[25697]: Reconfiguring daemon prometheus.a on smithi192 2024-04-02T04:58:00.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:57:59 smithi086 ceph-mon[29505]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:58:00.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:57:59 smithi086 ceph-mon[24811]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:58:00.334 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 04:58:00 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[25039]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:58:00] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T04:58:00.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:57:59 smithi192 ceph-mon[25697]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:58:02.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:01 smithi086 ceph-mon[24811]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T04:58:02.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:01 smithi086 ceph-mon[29505]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T04:58:02.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:01 smithi192 ceph-mon[25697]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T04:58:02.678 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:02 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[42606]: ts=2024-04-02T04:58:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: |\n Pool '{{ $labels.name }}' will be full in less than 5 days assuming the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed it's capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"172.21.15.86:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.192:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-02T04:58:04.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:03 smithi086 ceph-mon[24811]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-02T04:58:04.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:03 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:58:04.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:03 smithi086 ceph-mon[29505]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-02T04:58:04.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:03 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:58:04.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:03 smithi192 ceph-mon[25697]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-02T04:58:04.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:03 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:58:04.834 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 04:58:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[59010]: level=info ts=2024-04-02T04:58:04.553Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.003438197s 2024-04-02T04:58:05.428 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:05 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:58:05] "GET /metrics HTTP/1.1" 200 37897 "" "Prometheus/2.33.4" 2024-04-02T04:58:06.229 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:05 smithi192 ceph-mon[25697]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-02T04:58:06.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:05 smithi086 ceph-mon[24811]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-02T04:58:06.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:05 smithi086 ceph-mon[29505]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-02T04:58:07.678 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:07 smithi192 systemd[1]: Stopping Ceph prometheus.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T04:58:07.678 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:07 smithi192 bash[49108]: Error: no container with name or ID "ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus.a" found: no such container 2024-04-02T04:58:07.961 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:07 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[42606]: ts=2024-04-02T04:58:07.681Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-02T04:58:07.961 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:07 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[42606]: ts=2024-04-02T04:58:07.681Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-02T04:58:07.961 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:07 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[42606]: ts=2024-04-02T04:58:07.681Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-02T04:58:07.961 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:07 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[42606]: ts=2024-04-02T04:58:07.681Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-02T04:58:07.961 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:07 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[42606]: ts=2024-04-02T04:58:07.681Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-02T04:58:07.961 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:07 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[42606]: ts=2024-04-02T04:58:07.681Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-02T04:58:07.961 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:07 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[42606]: ts=2024-04-02T04:58:07.682Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-02T04:58:07.961 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:07 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[42606]: ts=2024-04-02T04:58:07.682Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-02T04:58:07.961 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:07 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[42606]: ts=2024-04-02T04:58:07.682Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-02T04:58:07.961 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:07 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[42606]: ts=2024-04-02T04:58:07.682Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-02T04:58:07.962 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:07 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[42606]: ts=2024-04-02T04:58:07.682Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-02T04:58:07.962 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:07 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[42606]: ts=2024-04-02T04:58:07.682Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-02T04:58:07.962 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:07 smithi192 podman[49121]: 2024-04-02 04:58:07.692044813 +0000 UTC m=+0.122702717 container died 782ae1d5e7f93611fd1424153fd849cd9e64b981a84f452dd70541a64d702187 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T04:58:08.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:07 smithi086 ceph-mon[29505]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-02T04:58:08.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:07 smithi086 ceph-mon[24811]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-02T04:58:08.311 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:07 smithi192 ceph-mon[25697]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-02T04:58:08.311 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:08 smithi192 podman[49121]: 2024-04-02 04:58:08.073642692 +0000 UTC m=+0.504300598 container cleanup 782ae1d5e7f93611fd1424153fd849cd9e64b981a84f452dd70541a64d702187 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T04:58:08.311 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:08 smithi192 bash[49121]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a 2024-04-02T04:58:08.311 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:08 smithi192 bash[49147]: Error: no container with name or ID "ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus.a" found: no such container 2024-04-02T04:58:08.658 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:08 smithi192 podman[49135]: 2024-04-02 04:58:08.308069966 +0000 UTC m=+0.619082904 container remove 782ae1d5e7f93611fd1424153fd849cd9e64b981a84f452dd70541a64d702187 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T04:58:08.659 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:08 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@prometheus.a.service: Deactivated successfully. 2024-04-02T04:58:08.659 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:08 smithi192 systemd[1]: Stopped Ceph prometheus.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T04:58:08.659 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:08 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@prometheus.a.service: Consumed 1.936s CPU time. 2024-04-02T04:58:08.659 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:08 smithi192 systemd[1]: Starting Ceph prometheus.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T04:58:08.659 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:08 smithi192 podman[49217]: 2024-04-02 04:58:08.440419125 +0000 UTC m=+0.018692769 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-02T04:58:08.659 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:08 smithi192 podman[49217]: 2024-04-02 04:58:08.551066056 +0000 UTC m=+0.129339742 container create d3ec96f3bde3c01aa626ae0c4f0b6f33a8f667fde5e21ecf40eb8b676e862459 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T04:58:08.929 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:08 smithi192 podman[49217]: 2024-04-02 04:58:08.736512169 +0000 UTC m=+0.314785812 container init d3ec96f3bde3c01aa626ae0c4f0b6f33a8f667fde5e21ecf40eb8b676e862459 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T04:58:08.929 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:08 smithi192 podman[49217]: 2024-04-02 04:58:08.739028462 +0000 UTC m=+0.317302108 container start d3ec96f3bde3c01aa626ae0c4f0b6f33a8f667fde5e21ecf40eb8b676e862459 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T04:58:08.929 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:08 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T04:58:08.763Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-02T04:58:08.929 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:08 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T04:58:08.763Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-02T04:58:08.929 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:08 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T04:58:08.763Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-02T04:58:08.929 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:08 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T04:58:08.763Z caller=main.go:518 level=info host_details="(Linux 5.14.0-434.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Mar 26 19:20:47 UTC 2024 x86_64 smithi192 (none))" 2024-04-02T04:58:08.929 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:08 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T04:58:08.763Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-02T04:58:08.929 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:08 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T04:58:08.763Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-02T04:58:08.929 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:08 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T04:58:08.765Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-02T04:58:08.929 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:08 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T04:58:08.765Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-02T04:58:08.930 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:08 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T04:58:08.766Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-02T04:58:08.930 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:08 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T04:58:08.769Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-02T04:58:08.930 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:08 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T04:58:08.769Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.759µs 2024-04-02T04:58:08.930 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:08 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T04:58:08.769Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-02T04:58:08.930 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:08 smithi192 bash[49217]: d3ec96f3bde3c01aa626ae0c4f0b6f33a8f667fde5e21ecf40eb8b676e862459 2024-04-02T04:58:08.930 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:08 smithi192 systemd[1]: Started Ceph prometheus.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T04:58:09.240 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:08 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T04:58:08.968Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-04-02T04:58:09.240 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:09 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T04:58:09.213Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-04-02T04:58:09.240 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:09 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T04:58:09.213Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-04-02T04:58:09.240 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:09 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T04:58:09.213Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=28.24µs wal_replay_duration=444.880783ms total_replay_duration=444.922863ms 2024-04-02T04:58:09.240 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:09 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T04:58:09.216Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-02T04:58:09.240 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:09 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T04:58:09.217Z caller=main.go:947 level=info msg="TSDB started" 2024-04-02T04:58:09.240 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:09 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T04:58:09.217Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-02T04:58:09.241 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:09 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T04:58:09.236Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=19.953366ms db_storage=688ns remote_storage=1.575µs web_handler=339ns query_engine=688ns scrape=402.413µs scrape_sd=85.188µs notify=10.494µs notify_sd=12.796µs rules=19.102813ms 2024-04-02T04:58:09.241 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:09 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T04:58:09.237Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-02T04:58:09.519 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:09 smithi192 systemd[1]: Stopping Ceph node-exporter.b for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T04:58:09.520 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:09 smithi192 bash[49343]: Error: no container with name or ID "ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter.b" found: no such container 2024-04-02T04:58:09.829 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:09 smithi192 podman[49356]: 2024-04-02 04:58:09.516503702 +0000 UTC m=+0.128582783 container died 27476f65f84fb08b5794840b29ebfe4e358583fd9ae5edde0e62121aef6240d1 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-02T04:58:10.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:09 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:10.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:09 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:10.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:09 smithi086 ceph-mon[29505]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-02T04:58:10.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:09 smithi086 ceph-mon[29505]: Reconfiguring daemon node-exporter.b on smithi192 2024-04-02T04:58:10.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:09 smithi086 ceph-mon[29505]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-02T04:58:10.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:09 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/3444249865' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T04:58:10.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:09 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1656494930' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/3128092578"}]: dispatch 2024-04-02T04:58:10.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:09 smithi086 ceph-mon[29505]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/3128092578"}]: dispatch 2024-04-02T04:58:10.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:09 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:10.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:09 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:10.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:09 smithi086 ceph-mon[24811]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-02T04:58:10.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:09 smithi086 ceph-mon[24811]: Reconfiguring daemon node-exporter.b on smithi192 2024-04-02T04:58:10.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:09 smithi086 ceph-mon[24811]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-02T04:58:10.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:09 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/3444249865' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T04:58:10.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:09 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1656494930' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/3128092578"}]: dispatch 2024-04-02T04:58:10.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:09 smithi086 ceph-mon[24811]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/3128092578"}]: dispatch 2024-04-02T04:58:10.113 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:09 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:10.114 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:09 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:10.114 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:09 smithi192 ceph-mon[25697]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-02T04:58:10.114 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:09 smithi192 ceph-mon[25697]: Reconfiguring daemon node-exporter.b on smithi192 2024-04-02T04:58:10.114 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:09 smithi192 ceph-mon[25697]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-02T04:58:10.114 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:09 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/3444249865' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T04:58:10.114 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:09 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1656494930' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/3128092578"}]: dispatch 2024-04-02T04:58:10.114 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:09 smithi192 ceph-mon[25697]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/3128092578"}]: dispatch 2024-04-02T04:58:10.114 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:09 smithi192 podman[49356]: 2024-04-02 04:58:09.848331544 +0000 UTC m=+0.460410615 container cleanup 27476f65f84fb08b5794840b29ebfe4e358583fd9ae5edde0e62121aef6240d1 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-02T04:58:10.115 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:09 smithi192 bash[49356]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b 2024-04-02T04:58:10.115 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 podman[49370]: 2024-04-02 04:58:10.040854537 +0000 UTC m=+0.529134922 container remove 27476f65f84fb08b5794840b29ebfe4e358583fd9ae5edde0e62121aef6240d1 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-02T04:58:10.115 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-02T04:58:10.115 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 bash[49383]: Error: no container with name or ID "ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter.b" found: no such container 2024-04-02T04:58:10.115 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-02T04:58:10.115 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 systemd[1]: Stopped Ceph node-exporter.b for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T04:58:10.115 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@node-exporter.b.service: Consumed 3.868s CPU time. 2024-04-02T04:58:10.115 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 systemd[1]: Starting Ceph node-exporter.b for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T04:58:10.416 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 podman[49460]: 2024-04-02 04:58:10.283419062 +0000 UTC m=+0.111928830 container create bd849d6c4fb0ce13ce45e53b9760a2e937f86e769246ed1b848aa4bf90eae0aa (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-02T04:58:10.416 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 podman[49460]: 2024-04-02 04:58:10.191120404 +0000 UTC m=+0.019630185 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-04-02T04:58:10.844 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 podman[49460]: 2024-04-02 04:58:10.510599358 +0000 UTC m=+0.339109124 container init bd849d6c4fb0ce13ce45e53b9760a2e937f86e769246ed1b848aa4bf90eae0aa (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-02T04:58:10.845 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 podman[49460]: 2024-04-02 04:58:10.513189625 +0000 UTC m=+0.341699403 container start bd849d6c4fb0ce13ce45e53b9760a2e937f86e769246ed1b848aa4bf90eae0aa (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-02T04:58:10.845 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.516Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-04-02T04:58:10.845 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.516Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-04-02T04:58:10.845 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.517Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-04-02T04:58:10.845 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.517Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-02T04:58:10.845 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.517Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-04-02T04:58:10.845 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.517Z caller=node_exporter.go:115 level=info collector=arp 2024-04-02T04:58:10.845 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.517Z caller=node_exporter.go:115 level=info collector=bcache 2024-04-02T04:58:10.845 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.517Z caller=node_exporter.go:115 level=info collector=bonding 2024-04-02T04:58:10.845 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.517Z caller=node_exporter.go:115 level=info collector=btrfs 2024-04-02T04:58:10.845 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.517Z caller=node_exporter.go:115 level=info collector=conntrack 2024-04-02T04:58:10.845 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=cpu 2024-04-02T04:58:10.845 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-04-02T04:58:10.846 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=diskstats 2024-04-02T04:58:10.846 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=dmi 2024-04-02T04:58:10.846 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=edac 2024-04-02T04:58:10.846 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=entropy 2024-04-02T04:58:10.846 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-04-02T04:58:10.846 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=filefd 2024-04-02T04:58:10.847 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=filesystem 2024-04-02T04:58:10.847 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=hwmon 2024-04-02T04:58:10.847 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=infiniband 2024-04-02T04:58:10.847 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=ipvs 2024-04-02T04:58:10.847 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=loadavg 2024-04-02T04:58:10.847 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=mdadm 2024-04-02T04:58:10.847 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=meminfo 2024-04-02T04:58:10.847 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=netclass 2024-04-02T04:58:10.847 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=netdev 2024-04-02T04:58:10.847 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=netstat 2024-04-02T04:58:10.847 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=nfs 2024-04-02T04:58:10.847 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=nfsd 2024-04-02T04:58:10.847 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=nvme 2024-04-02T04:58:10.847 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=os 2024-04-02T04:58:10.848 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-04-02T04:58:10.848 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=pressure 2024-04-02T04:58:10.848 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=rapl 2024-04-02T04:58:10.848 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=schedstat 2024-04-02T04:58:10.848 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=sockstat 2024-04-02T04:58:10.848 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=softnet 2024-04-02T04:58:10.848 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=stat 2024-04-02T04:58:10.848 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=tapestats 2024-04-02T04:58:10.848 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=textfile 2024-04-02T04:58:10.848 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-04-02T04:58:10.848 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=time 2024-04-02T04:58:10.848 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-04-02T04:58:10.848 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=uname 2024-04-02T04:58:10.848 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=vmstat 2024-04-02T04:58:10.849 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=xfs 2024-04-02T04:58:10.849 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:115 level=info collector=zfs 2024-04-02T04:58:10.849 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-04-02T04:58:10.849 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[49476]: ts=2024-04-02T04:58:10.518Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-04-02T04:58:10.849 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 bash[49460]: bd849d6c4fb0ce13ce45e53b9760a2e937f86e769246ed1b848aa4bf90eae0aa 2024-04-02T04:58:10.849 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 04:58:10 smithi192 systemd[1]: Started Ceph node-exporter.b for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T04:58:11.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-mon[25697]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/3128092578"}]': finished 2024-04-02T04:58:11.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-mon[25697]: osdmap e95: 8 total, 8 up, 8 in 2024-04-02T04:58:11.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1480581948' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/3680953997"}]: dispatch 2024-04-02T04:58:11.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:11.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:11.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:10 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:11.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:10 smithi086 ceph-mon[24811]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/3128092578"}]': finished 2024-04-02T04:58:11.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:10 smithi086 ceph-mon[24811]: osdmap e95: 8 total, 8 up, 8 in 2024-04-02T04:58:11.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:10 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1480581948' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/3680953997"}]: dispatch 2024-04-02T04:58:11.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:10 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:11.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:10 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:11.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:10 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:11.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:10 smithi086 ceph-mon[29505]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/3128092578"}]': finished 2024-04-02T04:58:11.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:10 smithi086 ceph-mon[29505]: osdmap e95: 8 total, 8 up, 8 in 2024-04-02T04:58:11.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:10 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1480581948' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/3680953997"}]: dispatch 2024-04-02T04:58:11.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:10 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:11.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:10 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:11.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:10 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:12.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:11 smithi192 ceph-mon[25697]: Reconfiguring grafana.a (dependencies changed)... 2024-04-02T04:58:12.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:11 smithi192 ceph-mon[25697]: Regenerating cephadm self-signed grafana TLS certificates 2024-04-02T04:58:12.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:11 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:12.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:11 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-02T04:58:12.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:11 smithi192 ceph-mon[25697]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-02T04:58:12.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:11 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1480581948' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/3680953997"}]': finished 2024-04-02T04:58:12.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:11 smithi192 ceph-mon[25697]: osdmap e96: 8 total, 8 up, 8 in 2024-04-02T04:58:12.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:11 smithi192 ceph-mon[25697]: Reconfiguring daemon grafana.a on smithi192 2024-04-02T04:58:12.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:11 smithi192 ceph-mon[25697]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:58:12.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:11 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1723224282' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/1542923653"}]: dispatch 2024-04-02T04:58:12.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:11 smithi086 ceph-mon[24811]: Reconfiguring grafana.a (dependencies changed)... 2024-04-02T04:58:12.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:11 smithi086 ceph-mon[24811]: Regenerating cephadm self-signed grafana TLS certificates 2024-04-02T04:58:12.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:11 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:12.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:11 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-02T04:58:12.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:11 smithi086 ceph-mon[24811]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-02T04:58:12.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:11 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1480581948' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/3680953997"}]': finished 2024-04-02T04:58:12.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:11 smithi086 ceph-mon[24811]: osdmap e96: 8 total, 8 up, 8 in 2024-04-02T04:58:12.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:11 smithi086 ceph-mon[24811]: Reconfiguring daemon grafana.a on smithi192 2024-04-02T04:58:12.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:11 smithi086 ceph-mon[24811]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:58:12.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:11 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1723224282' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/1542923653"}]: dispatch 2024-04-02T04:58:12.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:11 smithi086 ceph-mon[29505]: Reconfiguring grafana.a (dependencies changed)... 2024-04-02T04:58:12.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:11 smithi086 ceph-mon[29505]: Regenerating cephadm self-signed grafana TLS certificates 2024-04-02T04:58:12.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:11 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:12.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:11 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-02T04:58:12.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:11 smithi086 ceph-mon[29505]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-02T04:58:12.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:11 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1480581948' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/3680953997"}]': finished 2024-04-02T04:58:12.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:11 smithi086 ceph-mon[29505]: osdmap e96: 8 total, 8 up, 8 in 2024-04-02T04:58:12.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:11 smithi086 ceph-mon[29505]: Reconfiguring daemon grafana.a on smithi192 2024-04-02T04:58:12.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:11 smithi086 ceph-mon[29505]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:58:12.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:11 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1723224282' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/1542923653"}]: dispatch 2024-04-02T04:58:13.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:12 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1723224282' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/1542923653"}]': finished 2024-04-02T04:58:13.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:12 smithi192 ceph-mon[25697]: osdmap e97: 8 total, 8 up, 8 in 2024-04-02T04:58:13.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:12 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1691957944' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1206491125"}]: dispatch 2024-04-02T04:58:13.235 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:12 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1723224282' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/1542923653"}]': finished 2024-04-02T04:58:13.236 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:12 smithi086 ceph-mon[29505]: osdmap e97: 8 total, 8 up, 8 in 2024-04-02T04:58:13.236 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:12 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1691957944' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1206491125"}]: dispatch 2024-04-02T04:58:13.236 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:12 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1723224282' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/1542923653"}]': finished 2024-04-02T04:58:13.236 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:12 smithi086 ceph-mon[24811]: osdmap e97: 8 total, 8 up, 8 in 2024-04-02T04:58:13.236 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:12 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1691957944' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1206491125"}]: dispatch 2024-04-02T04:58:14.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:13 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1691957944' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1206491125"}]': finished 2024-04-02T04:58:14.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:13 smithi192 ceph-mon[25697]: osdmap e98: 8 total, 8 up, 8 in 2024-04-02T04:58:14.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:13 smithi192 ceph-mon[25697]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:58:14.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:13 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1741768281' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2034239143"}]: dispatch 2024-04-02T04:58:14.251 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:13 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1691957944' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1206491125"}]': finished 2024-04-02T04:58:14.251 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:13 smithi086 ceph-mon[29505]: osdmap e98: 8 total, 8 up, 8 in 2024-04-02T04:58:14.251 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:13 smithi086 ceph-mon[29505]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:58:14.251 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:13 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1741768281' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2034239143"}]: dispatch 2024-04-02T04:58:14.251 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:13 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1691957944' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1206491125"}]': finished 2024-04-02T04:58:14.251 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:13 smithi086 ceph-mon[24811]: osdmap e98: 8 total, 8 up, 8 in 2024-04-02T04:58:14.251 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:13 smithi086 ceph-mon[24811]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:58:14.251 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:13 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1741768281' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2034239143"}]: dispatch 2024-04-02T04:58:14.883 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:14 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1741768281' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2034239143"}]': finished 2024-04-02T04:58:14.883 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:14 smithi086 ceph-mon[29505]: osdmap e99: 8 total, 8 up, 8 in 2024-04-02T04:58:14.883 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:14 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/4197293741' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/1542923653"}]: dispatch 2024-04-02T04:58:14.883 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:14 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1741768281' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2034239143"}]': finished 2024-04-02T04:58:14.883 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:14 smithi086 ceph-mon[24811]: osdmap e99: 8 total, 8 up, 8 in 2024-04-02T04:58:14.884 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:14 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/4197293741' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/1542923653"}]: dispatch 2024-04-02T04:58:15.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:14 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1741768281' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2034239143"}]': finished 2024-04-02T04:58:15.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:14 smithi192 ceph-mon[25697]: osdmap e99: 8 total, 8 up, 8 in 2024-04-02T04:58:15.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:14 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/4197293741' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/1542923653"}]: dispatch 2024-04-02T04:58:16.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:15 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/4197293741' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/1542923653"}]': finished 2024-04-02T04:58:16.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:15 smithi192 ceph-mon[25697]: osdmap e100: 8 total, 8 up, 8 in 2024-04-02T04:58:16.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:15 smithi192 ceph-mon[25697]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:58:16.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:15 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1094044914' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1119175753"}]: dispatch 2024-04-02T04:58:16.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:15 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/4197293741' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/1542923653"}]': finished 2024-04-02T04:58:16.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:15 smithi086 ceph-mon[24811]: osdmap e100: 8 total, 8 up, 8 in 2024-04-02T04:58:16.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:15 smithi086 ceph-mon[24811]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:58:16.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:15 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1094044914' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1119175753"}]: dispatch 2024-04-02T04:58:16.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:15 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/4197293741' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/1542923653"}]': finished 2024-04-02T04:58:16.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:15 smithi086 ceph-mon[29505]: osdmap e100: 8 total, 8 up, 8 in 2024-04-02T04:58:16.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:15 smithi086 ceph-mon[29505]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:58:16.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:15 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1094044914' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1119175753"}]: dispatch 2024-04-02T04:58:17.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:16 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1094044914' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1119175753"}]': finished 2024-04-02T04:58:17.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:16 smithi192 ceph-mon[25697]: osdmap e101: 8 total, 8 up, 8 in 2024-04-02T04:58:17.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:16 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1094044914' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1119175753"}]': finished 2024-04-02T04:58:17.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:16 smithi086 ceph-mon[24811]: osdmap e101: 8 total, 8 up, 8 in 2024-04-02T04:58:17.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:16 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1094044914' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1119175753"}]': finished 2024-04-02T04:58:17.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:16 smithi086 ceph-mon[29505]: osdmap e101: 8 total, 8 up, 8 in 2024-04-02T04:58:17.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:58:17] "GET /metrics HTTP/1.1" 200 37897 "" "Prometheus/2.33.4" 2024-04-02T04:58:18.156 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:17 smithi192 ceph-mon[25697]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:58:18.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:17 smithi086 ceph-mon[24811]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:58:18.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:17 smithi086 ceph-mon[29505]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:58:18.678 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T04:58:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"smithi192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"172.21.15.192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-02T04:58:19.174 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:18 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:58:19.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:18 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:58:19.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:18 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:58:20.174 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:19 smithi192 ceph-mon[25697]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:58:20.174 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:19 smithi192 ceph-mon[25697]: from='client.24964 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:58:20.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:19 smithi086 ceph-mon[24811]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:58:20.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:19 smithi086 ceph-mon[24811]: from='client.24964 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:58:20.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:19 smithi086 ceph-mon[29505]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:58:20.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:19 smithi086 ceph-mon[29505]: from='client.24964 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:58:21.178 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:20 smithi192 systemd[1]: Stopping Ceph grafana.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T04:58:21.178 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:20 smithi192 bash[49757]: Error: no container with name or ID "ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana.a" found: no such container 2024-04-02T04:58:22.057 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:21 smithi192 ceph-mon[25697]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-02T04:58:22.057 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:21 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[41871]: t=2024-04-02T04:58:21+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-04-02T04:58:22.058 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:21 smithi192 podman[49770]: 2024-04-02 04:58:21.822421783 +0000 UTC m=+0.858804710 container died 6517aa8b3121ad60e5a023f694e54cf7d4154e7e7c0b98458231ad25b2d5732a (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a, com.redhat.component=ubi8-container, version=8.5, architecture=x86_64, io.buildah.version=1.24.2, vendor=Red Hat, Inc., description=Ceph Grafana Container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-03-28T10:36:18.413762, release=236.1648460182, distribution-scope=public, maintainer=Paul Cuzner , io.openshift.expose-services=, name=ubi8) 2024-04-02T04:58:22.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:21 smithi086 ceph-mon[24811]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-02T04:58:22.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:21 smithi086 ceph-mon[29505]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-02T04:58:22.428 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:22 smithi192 podman[49770]: 2024-04-02 04:58:22.194328529 +0000 UTC m=+1.230711452 container cleanup 6517aa8b3121ad60e5a023f694e54cf7d4154e7e7c0b98458231ad25b2d5732a (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a, io.openshift.expose-services=, architecture=x86_64, com.redhat.component=ubi8-container, description=Ceph Grafana Container, distribution-scope=public, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, version=8.5, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-03-28T10:36:18.413762, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.24.2, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, release=236.1648460182) 2024-04-02T04:58:22.428 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:22 smithi192 bash[49770]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a 2024-04-02T04:58:22.428 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:22 smithi192 bash[49805]: Error: no container with name or ID "ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana.a" found: no such container 2024-04-02T04:58:22.829 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:22 smithi192 podman[49789]: 2024-04-02 04:58:22.478702097 +0000 UTC m=+0.653615179 container remove 6517aa8b3121ad60e5a023f694e54cf7d4154e7e7c0b98458231ad25b2d5732a (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a, name=ubi8, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-03-28T10:36:18.413762, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, distribution-scope=public, release=236.1648460182, vcs-type=git, description=Ceph Grafana Container, io.openshift.expose-services=, io.buildah.version=1.24.2, maintainer=Paul Cuzner ) 2024-04-02T04:58:22.829 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:22 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@grafana.a.service: Deactivated successfully. 2024-04-02T04:58:22.829 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:22 smithi192 systemd[1]: Stopped Ceph grafana.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T04:58:22.829 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:22 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@grafana.a.service: Consumed 3.356s CPU time. 2024-04-02T04:58:22.829 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:22 smithi192 systemd[1]: Starting Ceph grafana.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T04:58:22.829 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:22 smithi192 podman[49873]: 2024-04-02 04:58:22.622697491 +0000 UTC m=+0.017539313 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-04-02T04:58:22.829 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:22 smithi192 podman[49873]: 2024-04-02 04:58:22.721273796 +0000 UTC m=+0.116115610 container create c1d8746fbef20d210c60b70b0493020f34cf4b54fa3895000ea5664a7243a4d9 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a, description=Ceph Grafana Container, maintainer=Paul Cuzner , version=8.5, io.openshift.expose-services=, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, vcs-type=git, architecture=x86_64, io.buildah.version=1.24.2, release=236.1648460182, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-03-28T10:36:18.413762, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public) 2024-04-02T04:58:23.126 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:22 smithi192 podman[49873]: 2024-04-02 04:58:22.931897595 +0000 UTC m=+0.326739408 container init c1d8746fbef20d210c60b70b0493020f34cf4b54fa3895000ea5664a7243a4d9 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, io.openshift.expose-services=, architecture=x86_64, maintainer=Paul Cuzner , build-date=2022-03-28T10:36:18.413762, name=ubi8, io.openshift.tags=base rhel8, release=236.1648460182, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, io.buildah.version=1.24.2, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, distribution-scope=public, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-04-02T04:58:23.126 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:22 smithi192 podman[49873]: 2024-04-02 04:58:22.935002429 +0000 UTC m=+0.329844243 container start c1d8746fbef20d210c60b70b0493020f34cf4b54fa3895000ea5664a7243a4d9 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a, release=236.1648460182, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, build-date=2022-03-28T10:36:18.413762, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, name=ubi8, io.openshift.tags=base rhel8, distribution-scope=public, io.buildah.version=1.24.2, summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, architecture=x86_64, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git) 2024-04-02T04:58:23.126 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:22 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[49888]: Failed to start grafana. error: failed to create log directory "/usr/share/grafana/data/log": mkdir /usr/share/grafana/data: permission denied 2024-04-02T04:58:23.126 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:22 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[49888]: failed to create log directory "/usr/share/grafana/data/log": mkdir /usr/share/grafana/data: permission denied 2024-04-02T04:58:23.126 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:23 smithi192 bash[49873]: c1d8746fbef20d210c60b70b0493020f34cf4b54fa3895000ea5664a7243a4d9 2024-04-02T04:58:23.126 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:23 smithi192 systemd[1]: Started Ceph grafana.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T04:58:23.126 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:23 smithi192 podman[49904]: 2024-04-02 04:58:23.040695794 +0000 UTC m=+0.072189891 container died c1d8746fbef20d210c60b70b0493020f34cf4b54fa3895000ea5664a7243a4d9 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, distribution-scope=public, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, release=236.1648460182, vcs-type=git, description=Ceph Grafana Container, version=8.5, build-date=2022-03-28T10:36:18.413762, io.openshift.expose-services=, architecture=x86_64, io.buildah.version=1.24.2) 2024-04-02T04:58:23.428 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:04:58:23] ENGINE Bus STOPPING 2024-04-02T04:58:23.428 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:04:58:23] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-02T04:58:23.428 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:04:58:23] ENGINE Bus STOPPED 2024-04-02T04:58:23.429 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:04:58:23] ENGINE Bus STARTING 2024-04-02T04:58:23.429 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:04:58:23] ENGINE Serving on http://:::9283 2024-04-02T04:58:23.429 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:04:58:23] ENGINE Bus STARTED 2024-04-02T04:58:23.429 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:04:58:23] ENGINE Bus STOPPING 2024-04-02T04:58:23.429 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:04:58:23] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-02T04:58:23.429 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:04:58:23] ENGINE Bus STOPPED 2024-04-02T04:58:23.429 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:04:58:23] ENGINE Bus STARTING 2024-04-02T04:58:23.429 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:04:58:23] ENGINE Serving on http://:::9283 2024-04-02T04:58:23.429 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:04:58:23] ENGINE Bus STARTED 2024-04-02T04:58:23.429 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:04:58:23] ENGINE Bus STOPPING 2024-04-02T04:58:23.429 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:04:58:23] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-02T04:58:23.429 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:04:58:23] ENGINE Bus STOPPED 2024-04-02T04:58:23.429 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:04:58:23] ENGINE Bus STARTING 2024-04-02T04:58:23.928 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:04:58:23] ENGINE Serving on http://:::9283 2024-04-02T04:58:23.928 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:23 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:04:58:23] ENGINE Bus STARTED 2024-04-02T04:58:23.929 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:23 smithi192 podman[49904]: 2024-04-02 04:58:23.6443873 +0000 UTC m=+0.675881383 container remove c1d8746fbef20d210c60b70b0493020f34cf4b54fa3895000ea5664a7243a4d9 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-03-28T10:36:18.413762, io.openshift.tags=base rhel8, release=236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, io.openshift.expose-services=, version=8.5, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, io.buildah.version=1.24.2, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., maintainer=Paul Cuzner , name=ubi8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container) 2024-04-02T04:58:23.929 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:23 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-02T04:58:23.929 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:23 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-02T04:58:24.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:24 smithi086 ceph-mon[29505]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 635 B/s rd, 0 op/s 2024-04-02T04:58:24.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:24 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:24.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:24 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:24.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:24 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T04:58:24.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:24 smithi086 ceph-mon[29505]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T04:58:24.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:24 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi086.front.sepia.ceph.com:9093"}]: dispatch 2024-04-02T04:58:24.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:24 smithi086 ceph-mon[29505]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi086.front.sepia.ceph.com:9093"}]: dispatch 2024-04-02T04:58:24.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:24 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:24.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:24 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T04:58:24.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:24 smithi086 ceph-mon[29505]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T04:58:24.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:24 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi192.front.sepia.ceph.com:9095"}]: dispatch 2024-04-02T04:58:24.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:24 smithi086 ceph-mon[29505]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi192.front.sepia.ceph.com:9095"}]: dispatch 2024-04-02T04:58:24.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:24 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:24.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:24 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T04:58:24.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:24 smithi086 ceph-mon[29505]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T04:58:24.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:24 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi192.front.sepia.ceph.com:3000"}]: dispatch 2024-04-02T04:58:24.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:24 smithi086 ceph-mon[29505]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi192.front.sepia.ceph.com:3000"}]: dispatch 2024-04-02T04:58:24.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:24 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:24.336 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:24 smithi086 ceph-mon[24811]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 635 B/s rd, 0 op/s 2024-04-02T04:58:24.336 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:24 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:24.336 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:24 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:24.336 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:24 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T04:58:24.336 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:24 smithi086 ceph-mon[24811]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T04:58:24.336 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:24 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi086.front.sepia.ceph.com:9093"}]: dispatch 2024-04-02T04:58:24.336 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:24 smithi086 ceph-mon[24811]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi086.front.sepia.ceph.com:9093"}]: dispatch 2024-04-02T04:58:24.336 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:24 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:24.336 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:24 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T04:58:24.336 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:24 smithi086 ceph-mon[24811]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T04:58:24.336 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:24 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi192.front.sepia.ceph.com:9095"}]: dispatch 2024-04-02T04:58:24.337 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:24 smithi086 ceph-mon[24811]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi192.front.sepia.ceph.com:9095"}]: dispatch 2024-04-02T04:58:24.337 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:24 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:24.337 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:24 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T04:58:24.337 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:24 smithi086 ceph-mon[24811]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T04:58:24.337 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:24 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi192.front.sepia.ceph.com:3000"}]: dispatch 2024-04-02T04:58:24.337 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:24 smithi086 ceph-mon[24811]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi192.front.sepia.ceph.com:3000"}]: dispatch 2024-04-02T04:58:24.337 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:24 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:24.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:24 smithi192 ceph-mon[25697]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 635 B/s rd, 0 op/s 2024-04-02T04:58:24.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:24 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:24.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:24 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:24.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:24 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T04:58:24.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:24 smithi192 ceph-mon[25697]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T04:58:24.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:24 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi086.front.sepia.ceph.com:9093"}]: dispatch 2024-04-02T04:58:24.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:24 smithi192 ceph-mon[25697]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi086.front.sepia.ceph.com:9093"}]: dispatch 2024-04-02T04:58:24.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:24 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:24.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:24 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T04:58:24.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:24 smithi192 ceph-mon[25697]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T04:58:24.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:24 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi192.front.sepia.ceph.com:9095"}]: dispatch 2024-04-02T04:58:24.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:24 smithi192 ceph-mon[25697]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi192.front.sepia.ceph.com:9095"}]: dispatch 2024-04-02T04:58:24.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:24 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:24.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:24 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T04:58:24.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:24 smithi192 ceph-mon[25697]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T04:58:24.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:24 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi192.front.sepia.ceph.com:3000"}]: dispatch 2024-04-02T04:58:24.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:24 smithi192 ceph-mon[25697]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi192.front.sepia.ceph.com:3000"}]: dispatch 2024-04-02T04:58:24.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:24 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:26.328 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:26 smithi086 ceph-mon[29505]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T04:58:26.328 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:26 smithi086 ceph-mon[24811]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T04:58:26.333 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:26 smithi192 ceph-mon[25697]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T04:58:27.642 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:27 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:58:27] "GET /metrics HTTP/1.1" 200 37889 "" "Prometheus/2.33.4" 2024-04-02T04:58:27.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:27 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:27.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:27 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:27.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:27 smithi086 ceph-mon[24811]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 927 B/s rd, 0 op/s 2024-04-02T04:58:27.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:27 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:27.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:27 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:27.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:27 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:58:27.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:27 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:58:27.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:27 smithi086 ceph-mon[24811]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1020 B/s rd, 0 op/s 2024-04-02T04:58:27.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:27 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:27.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:27 smithi086 ceph-mon[24811]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T04:58:27.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:27 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T04:58:27.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:27 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T04:58:27.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:27 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:58:27.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:27 smithi086 ceph-mon[24811]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T04:58:27.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:27 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:27.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:27 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:27.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:27 smithi086 ceph-mon[29505]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 927 B/s rd, 0 op/s 2024-04-02T04:58:27.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:27 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:27.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:27 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:27.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:27 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:58:27.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:27 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:58:27.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:27 smithi086 ceph-mon[29505]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1020 B/s rd, 0 op/s 2024-04-02T04:58:27.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:27 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:27.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:27 smithi086 ceph-mon[29505]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T04:58:27.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:27 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T04:58:27.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:27 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T04:58:27.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:27 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:58:27.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:27 smithi086 ceph-mon[29505]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T04:58:27.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:27 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:27.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:27 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:27.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:27 smithi192 ceph-mon[25697]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 927 B/s rd, 0 op/s 2024-04-02T04:58:27.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:27 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:27.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:27 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:27.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:27 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:58:27.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:27 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:58:27.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:27 smithi192 ceph-mon[25697]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1020 B/s rd, 0 op/s 2024-04-02T04:58:27.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:27 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:27.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:27 smithi192 ceph-mon[25697]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T04:58:27.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:27 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T04:58:27.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:27 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T04:58:27.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:27 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:58:27.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:27 smithi192 ceph-mon[25697]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T04:58:28.830 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:28 smithi086 ceph-mon[29505]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-02T04:58:28.831 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:28 smithi086 ceph-mon[24811]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-02T04:58:28.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:28 smithi192 ceph-mon[25697]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-02T04:58:28.928 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:28 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T04:58:28.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"smithi192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"172.21.15.192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-02T04:58:29.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:29 smithi086 ceph-mon[24811]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-04-02T04:58:29.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:29 smithi086 ceph-mon[29505]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-04-02T04:58:29.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:29 smithi192 ceph-mon[25697]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-04-02T04:58:30.651 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T04:58:30.245+0000 7f9e1a22d700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T04:58:30.652 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Creating ceph-iscsi config... 2024-04-02T04:58:30.652 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T04:58:30.652 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T04:58:30.652 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T04:58:30.652 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:58:30.652 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:58:30.652 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Traceback (most recent call last): 2024-04-02T04:58:30.652 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T04:58:30.652 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: return _run_code(code, main_globals, None, 2024-04-02T04:58:30.652 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T04:58:30.652 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: exec(code, run_globals) 2024-04-02T04:58:30.652 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T04:58:30.652 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T04:58:30.652 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T04:58:30.653 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T04:58:30.653 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T04:58:30.653 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T04:58:30.653 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T04:58:30.653 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:58:30.653 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:58:30.653 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Traceback (most recent call last): 2024-04-02T04:58:30.653 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T04:58:30.654 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T04:58:30.654 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T04:58:30.654 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T04:58:30.654 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T04:58:30.654 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: return self.event_loop.get_result(coro, timeout) 2024-04-02T04:58:30.654 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T04:58:30.654 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: return future.result(timeout) 2024-04-02T04:58:30.654 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T04:58:30.654 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: return self.__get_result() 2024-04-02T04:58:30.654 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T04:58:30.654 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: raise self._exception 2024-04-02T04:58:30.654 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T04:58:30.654 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: config_blobs=daemon_spec.final_config, 2024-04-02T04:58:30.654 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T04:58:30.654 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T04:58:30.655 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T04:58:30.655 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Creating ceph-iscsi config... 2024-04-02T04:58:30.655 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T04:58:30.655 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T04:58:30.655 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T04:58:30.655 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:58:30.655 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:58:30.655 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Traceback (most recent call last): 2024-04-02T04:58:30.655 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T04:58:30.655 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: return _run_code(code, main_globals, None, 2024-04-02T04:58:30.655 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T04:58:30.655 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: exec(code, run_globals) 2024-04-02T04:58:30.655 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T04:58:30.655 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T04:58:30.655 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T04:58:30.656 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T04:58:30.656 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T04:58:30.656 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T04:58:30.656 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T04:58:30.656 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:58:30.656 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:58:30.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:30 smithi192 ceph-mon[25697]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T04:58:30.929 INFO:journalctl@ceph.mon.b.smithi192.stdout: Creating ceph-iscsi config... 2024-04-02T04:58:30.929 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T04:58:30.929 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T04:58:30.929 INFO:journalctl@ceph.mon.b.smithi192.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T04:58:30.929 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:58:30.929 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:58:30.929 INFO:journalctl@ceph.mon.b.smithi192.stdout: Traceback (most recent call last): 2024-04-02T04:58:30.929 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T04:58:30.929 INFO:journalctl@ceph.mon.b.smithi192.stdout: return _run_code(code, main_globals, None, 2024-04-02T04:58:30.929 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T04:58:30.930 INFO:journalctl@ceph.mon.b.smithi192.stdout: exec(code, run_globals) 2024-04-02T04:58:30.930 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T04:58:30.930 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T04:58:30.930 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T04:58:30.930 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T04:58:30.930 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T04:58:30.930 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T04:58:30.930 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T04:58:30.930 INFO:journalctl@ceph.mon.b.smithi192.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:58:30.930 INFO:journalctl@ceph.mon.b.smithi192.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:58:30.930 INFO:journalctl@ceph.mon.b.smithi192.stdout: Traceback (most recent call last): 2024-04-02T04:58:30.931 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T04:58:30.931 INFO:journalctl@ceph.mon.b.smithi192.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T04:58:30.931 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T04:58:30.931 INFO:journalctl@ceph.mon.b.smithi192.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T04:58:30.931 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T04:58:30.931 INFO:journalctl@ceph.mon.b.smithi192.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-02T04:58:30.931 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T04:58:30.931 INFO:journalctl@ceph.mon.b.smithi192.stdout: return future.result(timeout) 2024-04-02T04:58:30.931 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T04:58:30.931 INFO:journalctl@ceph.mon.b.smithi192.stdout: return self.__get_result() 2024-04-02T04:58:30.931 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T04:58:30.931 INFO:journalctl@ceph.mon.b.smithi192.stdout: raise self._exception 2024-04-02T04:58:30.931 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T04:58:30.931 INFO:journalctl@ceph.mon.b.smithi192.stdout: config_blobs=daemon_spec.final_config, 2024-04-02T04:58:30.932 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T04:58:30.932 INFO:journalctl@ceph.mon.b.smithi192.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T04:58:30.932 INFO:journalctl@ceph.mon.b.smithi192.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T04:58:30.932 INFO:journalctl@ceph.mon.b.smithi192.stdout: Creating ceph-iscsi config... 2024-04-02T04:58:30.932 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T04:58:30.932 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T04:58:30.932 INFO:journalctl@ceph.mon.b.smithi192.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T04:58:30.932 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:58:30.932 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:58:30.932 INFO:journalctl@ceph.mon.b.smithi192.stdout: Traceback (most recent call last): 2024-04-02T04:58:30.932 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T04:58:30.932 INFO:journalctl@ceph.mon.b.smithi192.stdout: return _run_code(code, main_globals, None, 2024-04-02T04:58:30.932 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T04:58:30.933 INFO:journalctl@ceph.mon.b.smithi192.stdout: exec(code, run_globals) 2024-04-02T04:58:30.933 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T04:58:30.933 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T04:58:30.933 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T04:58:30.933 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T04:58:30.933 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T04:58:30.933 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T04:58:30.933 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T04:58:30.933 INFO:journalctl@ceph.mon.b.smithi192.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:58:30.933 INFO:journalctl@ceph.mon.b.smithi192.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:58:31.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:30 smithi086 ceph-mon[24811]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T04:58:31.084 INFO:journalctl@ceph.mon.a.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T04:58:31.084 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T04:58:31.084 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T04:58:31.084 INFO:journalctl@ceph.mon.a.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T04:58:31.084 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:58:31.084 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:58:31.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: Traceback (most recent call last): 2024-04-02T04:58:31.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T04:58:31.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T04:58:31.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T04:58:31.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: exec(code, run_globals) 2024-04-02T04:58:31.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T04:58:31.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T04:58:31.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T04:58:31.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T04:58:31.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T04:58:31.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T04:58:31.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T04:58:31.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:58:31.086 INFO:journalctl@ceph.mon.a.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:58:31.086 INFO:journalctl@ceph.mon.a.smithi086.stdout: Traceback (most recent call last): 2024-04-02T04:58:31.086 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T04:58:31.086 INFO:journalctl@ceph.mon.a.smithi086.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T04:58:31.086 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T04:58:31.086 INFO:journalctl@ceph.mon.a.smithi086.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T04:58:31.086 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T04:58:31.086 INFO:journalctl@ceph.mon.a.smithi086.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-02T04:58:31.086 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T04:58:31.086 INFO:journalctl@ceph.mon.a.smithi086.stdout: return future.result(timeout) 2024-04-02T04:58:31.087 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T04:58:31.087 INFO:journalctl@ceph.mon.a.smithi086.stdout: return self.__get_result() 2024-04-02T04:58:31.087 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T04:58:31.087 INFO:journalctl@ceph.mon.a.smithi086.stdout: raise self._exception 2024-04-02T04:58:31.087 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T04:58:31.088 INFO:journalctl@ceph.mon.a.smithi086.stdout: config_blobs=daemon_spec.final_config, 2024-04-02T04:58:31.088 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T04:58:31.088 INFO:journalctl@ceph.mon.a.smithi086.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T04:58:31.088 INFO:journalctl@ceph.mon.a.smithi086.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T04:58:31.088 INFO:journalctl@ceph.mon.a.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T04:58:31.088 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T04:58:31.088 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T04:58:31.088 INFO:journalctl@ceph.mon.a.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T04:58:31.089 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:58:31.089 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:58:31.089 INFO:journalctl@ceph.mon.a.smithi086.stdout: Traceback (most recent call last): 2024-04-02T04:58:31.089 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T04:58:31.089 INFO:journalctl@ceph.mon.a.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T04:58:31.089 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T04:58:31.089 INFO:journalctl@ceph.mon.a.smithi086.stdout: exec(code, run_globals) 2024-04-02T04:58:31.089 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T04:58:31.089 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T04:58:31.089 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T04:58:31.089 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T04:58:31.089 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T04:58:31.089 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T04:58:31.089 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T04:58:31.089 INFO:journalctl@ceph.mon.a.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:58:31.090 INFO:journalctl@ceph.mon.a.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:58:31.090 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:30 smithi086 ceph-mon[29505]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T04:58:31.090 INFO:journalctl@ceph.mon.c.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T04:58:31.090 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T04:58:31.090 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T04:58:31.090 INFO:journalctl@ceph.mon.c.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T04:58:31.090 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:58:31.090 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:58:31.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: Traceback (most recent call last): 2024-04-02T04:58:31.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T04:58:31.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T04:58:31.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T04:58:31.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: exec(code, run_globals) 2024-04-02T04:58:31.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T04:58:31.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T04:58:31.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T04:58:31.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T04:58:31.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T04:58:31.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T04:58:31.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T04:58:31.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:58:31.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:58:31.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: Traceback (most recent call last): 2024-04-02T04:58:31.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T04:58:31.092 INFO:journalctl@ceph.mon.c.smithi086.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T04:58:31.092 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T04:58:31.092 INFO:journalctl@ceph.mon.c.smithi086.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T04:58:31.092 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T04:58:31.092 INFO:journalctl@ceph.mon.c.smithi086.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-02T04:58:31.092 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T04:58:31.092 INFO:journalctl@ceph.mon.c.smithi086.stdout: return future.result(timeout) 2024-04-02T04:58:31.092 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T04:58:31.092 INFO:journalctl@ceph.mon.c.smithi086.stdout: return self.__get_result() 2024-04-02T04:58:31.092 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T04:58:31.092 INFO:journalctl@ceph.mon.c.smithi086.stdout: raise self._exception 2024-04-02T04:58:31.092 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T04:58:31.092 INFO:journalctl@ceph.mon.c.smithi086.stdout: config_blobs=daemon_spec.final_config, 2024-04-02T04:58:31.092 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T04:58:31.092 INFO:journalctl@ceph.mon.c.smithi086.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T04:58:31.093 INFO:journalctl@ceph.mon.c.smithi086.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T04:58:31.093 INFO:journalctl@ceph.mon.c.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T04:58:31.093 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T04:58:31.093 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T04:58:31.093 INFO:journalctl@ceph.mon.c.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T04:58:31.093 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:58:31.093 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:58:31.093 INFO:journalctl@ceph.mon.c.smithi086.stdout: Traceback (most recent call last): 2024-04-02T04:58:31.093 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T04:58:31.093 INFO:journalctl@ceph.mon.c.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T04:58:31.093 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T04:58:31.093 INFO:journalctl@ceph.mon.c.smithi086.stdout: exec(code, run_globals) 2024-04-02T04:58:31.093 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T04:58:31.093 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T04:58:31.093 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T04:58:31.094 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T04:58:31.094 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T04:58:31.094 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T04:58:31.094 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T04:58:31.094 INFO:journalctl@ceph.mon.c.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:58:31.094 INFO:journalctl@ceph.mon.c.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:58:32.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:31 smithi086 ceph-mon[24811]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T04:58:32.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:31 smithi086 ceph-mon[29505]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T04:58:32.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:31 smithi192 ceph-mon[25697]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T04:58:33.946 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:33 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@grafana.a.service: Scheduled restart job, restart counter is at 1. 2024-04-02T04:58:33.946 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:33 smithi192 systemd[1]: Stopped Ceph grafana.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T04:58:33.946 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:33 smithi192 systemd[1]: Starting Ceph grafana.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T04:58:33.946 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:33 smithi192 podman[50560]: 2024-04-02 04:58:33.81305413 +0000 UTC m=+0.017747440 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-04-02T04:58:34.248 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:34 smithi192 ceph-mon[25697]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T04:58:34.248 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:34 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:34.249 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:34 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:58:34.249 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:33 smithi192 podman[50560]: 2024-04-02 04:58:33.94316695 +0000 UTC m=+0.147860266 container create 535dad3336a817fa35656a8a595ba2371e9b346947f965098c95ff25708bf67f (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a, architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, name=ubi8, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, build-date=2022-03-28T10:36:18.413762, description=Ceph Grafana Container, io.openshift.expose-services=, io.buildah.version=1.24.2, release=236.1648460182, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , distribution-scope=public, version=8.5, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-02T04:58:34.249 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:34 smithi192 podman[50560]: 2024-04-02 04:58:34.16247422 +0000 UTC m=+0.367167527 container init 535dad3336a817fa35656a8a595ba2371e9b346947f965098c95ff25708bf67f (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, name=ubi8, description=Ceph Grafana Container, version=8.5, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, distribution-scope=public, build-date=2022-03-28T10:36:18.413762, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, io.buildah.version=1.24.2, architecture=x86_64) 2024-04-02T04:58:34.249 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:34 smithi192 podman[50560]: 2024-04-02 04:58:34.165669505 +0000 UTC m=+0.370362815 container start 535dad3336a817fa35656a8a595ba2371e9b346947f965098c95ff25708bf67f (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-03-28T10:36:18.413762, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, release=236.1648460182, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, io.buildah.version=1.24.2, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, com.redhat.component=ubi8-container, maintainer=Paul Cuzner , vcs-type=git, io.openshift.expose-services=, distribution-scope=public, description=Ceph Grafana Container) 2024-04-02T04:58:34.249 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:34 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[50575]: Failed to start grafana. error: failed to create log directory "/usr/share/grafana/data/log": mkdir /usr/share/grafana/data: permission denied 2024-04-02T04:58:34.249 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:34 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[50575]: failed to create log directory "/usr/share/grafana/data/log": mkdir /usr/share/grafana/data: permission denied 2024-04-02T04:58:34.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:34 smithi086 ceph-mon[24811]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T04:58:34.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:34 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:34.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:34 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:58:34.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:34 smithi086 ceph-mon[29505]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T04:58:34.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:34 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:58:34.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:34 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:58:34.678 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:34 smithi192 bash[50560]: 535dad3336a817fa35656a8a595ba2371e9b346947f965098c95ff25708bf67f 2024-04-02T04:58:34.678 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:34 smithi192 systemd[1]: Started Ceph grafana.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T04:58:34.678 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:34 smithi192 podman[50591]: 2024-04-02 04:58:34.270399617 +0000 UTC m=+0.072204103 container died 535dad3336a817fa35656a8a595ba2371e9b346947f965098c95ff25708bf67f (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a, description=Ceph Grafana Container, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, io.openshift.expose-services=, release=236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, name=ubi8, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.24.2, architecture=x86_64, build-date=2022-03-28T10:36:18.413762, maintainer=Paul Cuzner , io.openshift.tags=base rhel8, distribution-scope=public) 2024-04-02T04:58:35.179 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:34 smithi192 podman[50591]: 2024-04-02 04:58:34.874410611 +0000 UTC m=+0.676215085 container remove 535dad3336a817fa35656a8a595ba2371e9b346947f965098c95ff25708bf67f (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a, name=ubi8, vcs-type=git, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.24.2, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, description=Ceph Grafana Container, release=236.1648460182, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, maintainer=Paul Cuzner , build-date=2022-03-28T10:36:18.413762, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8) 2024-04-02T04:58:35.179 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:34 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-02T04:58:35.179 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:34 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-02T04:58:36.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:36 smithi086 ceph-mon[24811]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 612 B/s rd, 0 op/s 2024-04-02T04:58:36.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:36 smithi086 ceph-mon[29505]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 612 B/s rd, 0 op/s 2024-04-02T04:58:36.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:36 smithi192 ceph-mon[25697]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 612 B/s rd, 0 op/s 2024-04-02T04:58:37.446 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:58:37] "GET /metrics HTTP/1.1" 200 37970 "" "Prometheus/2.33.4" 2024-04-02T04:58:38.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:38 smithi086 ceph-mon[29505]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 612 B/s rd, 0 op/s 2024-04-02T04:58:38.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:38 smithi086 ceph-mon[24811]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 612 B/s rd, 0 op/s 2024-04-02T04:58:38.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:38 smithi192 ceph-mon[25697]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 612 B/s rd, 0 op/s 2024-04-02T04:58:38.928 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:38 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T04:58:38.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"smithi192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"172.21.15.192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-02T04:58:40.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:40 smithi086 ceph-mon[24811]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-02T04:58:40.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:40 smithi086 ceph-mon[29505]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-02T04:58:40.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:40 smithi192 ceph-mon[25697]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-02T04:58:42.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:41 smithi086 ceph-mon[24811]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:58:42.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:41 smithi086 ceph-mon[29505]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:58:42.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:41 smithi192 ceph-mon[25697]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:58:43.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:42 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/3349679023' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T04:58:43.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:42 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/3349679023' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T04:58:43.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:42 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/3349679023' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T04:58:44.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:43 smithi086 ceph-mon[29505]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:58:44.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:43 smithi086 ceph-mon[24811]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:58:44.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:43 smithi192 ceph-mon[25697]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:58:45.186 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:44 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@grafana.a.service: Scheduled restart job, restart counter is at 2. 2024-04-02T04:58:45.186 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:44 smithi192 systemd[1]: Stopped Ceph grafana.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T04:58:45.186 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:44 smithi192 systemd[1]: Starting Ceph grafana.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T04:58:45.186 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:45 smithi192 podman[50659]: 2024-04-02 04:58:45.062378955 +0000 UTC m=+0.017687241 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-04-02T04:58:45.480 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:45 smithi192 podman[50659]: 2024-04-02 04:58:45.18330664 +0000 UTC m=+0.138614912 container create 295eca324cc3f0ccc3e49a91e227c178038546ab7d9c4405d676982bbccec226 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, version=8.5, io.openshift.tags=base rhel8, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=236.1648460182, com.redhat.component=ubi8-container, distribution-scope=public, build-date=2022-03-28T10:36:18.413762, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.24.2, maintainer=Paul Cuzner , name=ubi8) 2024-04-02T04:58:45.480 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:45 smithi192 podman[50659]: 2024-04-02 04:58:45.393998248 +0000 UTC m=+0.349306533 container init 295eca324cc3f0ccc3e49a91e227c178038546ab7d9c4405d676982bbccec226 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, io.buildah.version=1.24.2, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, distribution-scope=public, build-date=2022-03-28T10:36:18.413762, name=ubi8, release=236.1648460182, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-04-02T04:58:45.481 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:45 smithi192 podman[50659]: 2024-04-02 04:58:45.396636229 +0000 UTC m=+0.351944508 container start 295eca324cc3f0ccc3e49a91e227c178038546ab7d9c4405d676982bbccec226 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a, version=8.5, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, architecture=x86_64, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, description=Ceph Grafana Container, com.redhat.component=ubi8-container, name=ubi8, io.buildah.version=1.24.2, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=236.1648460182, vendor=Red Hat, Inc., io.openshift.expose-services=, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, build-date=2022-03-28T10:36:18.413762, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-04-02T04:58:45.481 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:45 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[50674]: Failed to start grafana. error: failed to create log directory "/usr/share/grafana/data/log": mkdir /usr/share/grafana/data: permission denied 2024-04-02T04:58:45.481 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:45 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[50674]: failed to create log directory "/usr/share/grafana/data/log": mkdir /usr/share/grafana/data: permission denied 2024-04-02T04:58:45.481 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:45 smithi192 bash[50659]: 295eca324cc3f0ccc3e49a91e227c178038546ab7d9c4405d676982bbccec226 2024-04-02T04:58:45.885 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:45 smithi192 systemd[1]: Started Ceph grafana.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T04:58:45.885 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:45 smithi192 podman[50690]: 2024-04-02 04:58:45.501798781 +0000 UTC m=+0.071971703 container died 295eca324cc3f0ccc3e49a91e227c178038546ab7d9c4405d676982bbccec226 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a, io.buildah.version=1.24.2, distribution-scope=public, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, version=8.5, description=Ceph Grafana Container, io.openshift.tags=base rhel8, name=ubi8, architecture=x86_64, build-date=2022-03-28T10:36:18.413762, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, release=236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git) 2024-04-02T04:58:46.136 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:46 smithi192 ceph-mon[25697]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-02T04:58:46.136 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:46 smithi192 podman[50690]: 2024-04-02 04:58:46.114484169 +0000 UTC m=+0.684657088 container remove 295eca324cc3f0ccc3e49a91e227c178038546ab7d9c4405d676982bbccec226 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.24.2, io.openshift.tags=base rhel8, name=ubi8, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, maintainer=Paul Cuzner , architecture=x86_64, release=236.1648460182, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, build-date=2022-03-28T10:36:18.413762, version=8.5, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-02T04:58:46.136 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:46 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-02T04:58:46.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:46 smithi086 ceph-mon[29505]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-02T04:58:46.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:46 smithi086 ceph-mon[24811]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-02T04:58:46.428 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:46 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-02T04:58:47.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:58:47] "GET /metrics HTTP/1.1" 200 37970 "" "Prometheus/2.33.4" 2024-04-02T04:58:48.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:48 smithi086 ceph-mon[24811]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-02T04:58:48.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:48 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:58:48.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:48 smithi086 ceph-mon[29505]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-02T04:58:48.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:48 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:58:48.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:48 smithi192 ceph-mon[25697]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-02T04:58:48.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:48 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:58:48.928 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:48 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T04:58:48.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"smithi192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"172.21.15.192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-02T04:58:50.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:50 smithi086 ceph-mon[24811]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-02T04:58:50.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:50 smithi086 ceph-mon[29505]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-02T04:58:50.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:50 smithi192 ceph-mon[25697]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-02T04:58:51.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:51 smithi086 ceph-mon[24811]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-02T04:58:51.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:51 smithi086 ceph-mon[29505]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-02T04:58:52.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:51 smithi192 ceph-mon[25697]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-02T04:58:53.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:52 smithi086 ceph-mon[24811]: from='client.15132 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:58:53.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:52 smithi086 ceph-mon[29505]: from='client.15132 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:58:53.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:52 smithi192 ceph-mon[25697]: from='client.15132 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:58:54.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:53 smithi086 ceph-mon[29505]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-02T04:58:54.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:53 smithi086 ceph-mon[24811]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-02T04:58:54.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:53 smithi192 ceph-mon[25697]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-02T04:58:56.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:56 smithi086 ceph-mon[24811]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T04:58:56.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:56 smithi086 ceph-mon[29505]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T04:58:56.416 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:56 smithi192 ceph-mon[25697]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T04:58:56.416 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:56 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@grafana.a.service: Scheduled restart job, restart counter is at 3. 2024-04-02T04:58:56.416 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:56 smithi192 systemd[1]: Stopped Ceph grafana.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T04:58:56.416 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:56 smithi192 systemd[1]: Starting Ceph grafana.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T04:58:56.678 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:56 smithi192 podman[50772]: 2024-04-02 04:58:56.315014362 +0000 UTC m=+0.017354242 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-04-02T04:58:56.678 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:56 smithi192 podman[50772]: 2024-04-02 04:58:56.601800704 +0000 UTC m=+0.304140571 container create 65d5ac46a24406ae5b16a107774d528a72dc3f34c95f694d7900607aae109771 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-03-28T10:36:18.413762, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, name=ubi8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.24.2, release=236.1648460182, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, maintainer=Paul Cuzner , vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, version=8.5, distribution-scope=public, com.redhat.component=ubi8-container, architecture=x86_64) 2024-04-02T04:58:57.179 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:56 smithi192 podman[50772]: 2024-04-02 04:58:56.812589046 +0000 UTC m=+0.514928924 container init 65d5ac46a24406ae5b16a107774d528a72dc3f34c95f694d7900607aae109771 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-03-28T10:36:18.413762, description=Ceph Grafana Container, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, io.buildah.version=1.24.2, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, maintainer=Paul Cuzner , version=8.5, distribution-scope=public, release=236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, architecture=x86_64, vcs-type=git) 2024-04-02T04:58:57.179 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:56 smithi192 podman[50772]: 2024-04-02 04:58:56.815281734 +0000 UTC m=+0.517621601 container start 65d5ac46a24406ae5b16a107774d528a72dc3f34c95f694d7900607aae109771 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a, vendor=Red Hat, Inc., io.buildah.version=1.24.2, vcs-type=git, name=ubi8, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, build-date=2022-03-28T10:36:18.413762, description=Ceph Grafana Container, version=8.5, io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=236.1648460182, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.openshift.tags=base rhel8) 2024-04-02T04:58:57.179 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:56 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[50787]: Failed to start grafana. error: failed to create log directory "/usr/share/grafana/data/log": mkdir /usr/share/grafana/data: permission denied 2024-04-02T04:58:57.179 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:56 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[50787]: failed to create log directory "/usr/share/grafana/data/log": mkdir /usr/share/grafana/data: permission denied 2024-04-02T04:58:57.179 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:56 smithi192 bash[50772]: 65d5ac46a24406ae5b16a107774d528a72dc3f34c95f694d7900607aae109771 2024-04-02T04:58:57.179 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:56 smithi192 systemd[1]: Started Ceph grafana.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T04:58:57.179 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:56 smithi192 podman[50803]: 2024-04-02 04:58:56.921318814 +0000 UTC m=+0.072416728 container died 65d5ac46a24406ae5b16a107774d528a72dc3f34c95f694d7900607aae109771 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, description=Ceph Grafana Container, architecture=x86_64, version=8.5, io.openshift.tags=base rhel8, io.k8s.display-name=Red Hat Universal Base Image 8, build-date=2022-03-28T10:36:18.413762, name=ubi8, release=236.1648460182, vcs-type=git, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.24.2, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-04-02T04:58:57.494 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:58:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:58:57] "GET /metrics HTTP/1.1" 200 37968 "" "Prometheus/2.33.4" 2024-04-02T04:58:57.928 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:57 smithi192 podman[50803]: 2024-04-02 04:58:57.491280904 +0000 UTC m=+0.642378805 container remove 65d5ac46a24406ae5b16a107774d528a72dc3f34c95f694d7900607aae109771 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.tags=base rhel8, release=236.1648460182, io.buildah.version=1.24.2, description=Ceph Grafana Container, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, name=ubi8, build-date=2022-03-28T10:36:18.413762, maintainer=Paul Cuzner , vcs-type=git, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-02T04:58:57.928 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:57 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-02T04:58:57.928 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:58:57 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-02T04:58:58.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:58:58 smithi086 ceph-mon[24811]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:58:58.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:58:58 smithi086 ceph-mon[29505]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:58:58.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:58:58 smithi192 ceph-mon[25697]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:58:58.928 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:58:58 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T04:58:58.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"smithi192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"172.21.15.192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-02T04:59:00.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:00 smithi086 ceph-mon[24811]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:59:00.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:00 smithi086 ceph-mon[29505]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:59:00.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:00 smithi192 ceph-mon[25697]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:59:01.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:01 smithi086 ceph-mon[24811]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:59:01.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:01 smithi086 ceph-mon[29505]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:59:02.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:01 smithi192 ceph-mon[25697]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:59:03.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:02 smithi086 ceph-mon[24811]: from='client.15132 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:59:03.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:02 smithi086 ceph-mon[29505]: from='client.15132 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:59:03.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:02 smithi192 ceph-mon[25697]: from='client.15132 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:59:04.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:03 smithi086 ceph-mon[29505]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:59:04.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:03 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:59:04.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:03 smithi086 ceph-mon[24811]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:59:04.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:03 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:59:04.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:03 smithi192 ceph-mon[25697]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:59:04.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:03 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:59:06.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:06 smithi086 ceph-mon[24811]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:59:06.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:06 smithi086 ceph-mon[29505]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:59:06.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:06 smithi192 ceph-mon[25697]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:59:07.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:07 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:59:07] "GET /metrics HTTP/1.1" 200 37965 "" "Prometheus/2.33.4" 2024-04-02T04:59:08.016 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:59:07 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@grafana.a.service: Scheduled restart job, restart counter is at 4. 2024-04-02T04:59:08.016 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:59:07 smithi192 systemd[1]: Stopped Ceph grafana.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T04:59:08.016 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:59:07 smithi192 systemd[1]: Starting Ceph grafana.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T04:59:08.016 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:59:07 smithi192 podman[50875]: 2024-04-02 04:59:07.808504872 +0000 UTC m=+0.017822113 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-04-02T04:59:08.016 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:59:07 smithi192 podman[50875]: 2024-04-02 04:59:07.908258865 +0000 UTC m=+0.117576108 container create 7e80fc644c048b2903442d71b114ea0fe61eb5b05c2788d0e2156c3cd0160ff9 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a, maintainer=Paul Cuzner , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, description=Ceph Grafana Container, name=ubi8, com.redhat.component=ubi8-container, distribution-scope=public, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.24.2, vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-03-28T10:36:18.413762, release=236.1648460182, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, vendor=Red Hat, Inc., version=8.5) 2024-04-02T04:59:08.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:08 smithi086 ceph-mon[24811]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:59:08.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:08 smithi086 ceph-mon[29505]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:59:08.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:08 smithi192 ceph-mon[25697]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:59:08.428 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:59:08 smithi192 podman[50875]: 2024-04-02 04:59:08.143882381 +0000 UTC m=+0.353199621 container init 7e80fc644c048b2903442d71b114ea0fe61eb5b05c2788d0e2156c3cd0160ff9 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a, description=Ceph Grafana Container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.24.2, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, vendor=Red Hat, Inc., io.openshift.expose-services=, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, build-date=2022-03-28T10:36:18.413762, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.k8s.display-name=Red Hat Universal Base Image 8, release=236.1648460182) 2024-04-02T04:59:08.428 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:59:08 smithi192 podman[50875]: 2024-04-02 04:59:08.146853532 +0000 UTC m=+0.356170776 container start 7e80fc644c048b2903442d71b114ea0fe61eb5b05c2788d0e2156c3cd0160ff9 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., io.openshift.expose-services=, architecture=x86_64, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.tags=base rhel8, description=Ceph Grafana Container, name=ubi8, version=8.5, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=236.1648460182, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.24.2, build-date=2022-03-28T10:36:18.413762, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-04-02T04:59:08.428 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:59:08 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[50889]: Failed to start grafana. error: failed to create log directory "/usr/share/grafana/data/log": mkdir /usr/share/grafana/data: permission denied 2024-04-02T04:59:08.429 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:59:08 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[50889]: failed to create log directory "/usr/share/grafana/data/log": mkdir /usr/share/grafana/data: permission denied 2024-04-02T04:59:08.429 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:59:08 smithi192 bash[50875]: 7e80fc644c048b2903442d71b114ea0fe61eb5b05c2788d0e2156c3cd0160ff9 2024-04-02T04:59:08.429 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:59:08 smithi192 systemd[1]: Started Ceph grafana.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T04:59:08.429 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:59:08 smithi192 podman[50905]: 2024-04-02 04:59:08.251466865 +0000 UTC m=+0.071907853 container died 7e80fc644c048b2903442d71b114ea0fe61eb5b05c2788d0e2156c3cd0160ff9 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, name=ubi8, io.openshift.expose-services=, architecture=x86_64, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.24.2, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , build-date=2022-03-28T10:36:18.413762, distribution-scope=public, release=236.1648460182) 2024-04-02T04:59:08.842 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:59:08 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T04:59:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"smithi192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"172.21.15.192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-02T04:59:09.178 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:59:08 smithi192 podman[50905]: 2024-04-02 04:59:08.839572167 +0000 UTC m=+0.660013145 container remove 7e80fc644c048b2903442d71b114ea0fe61eb5b05c2788d0e2156c3cd0160ff9 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, io.openshift.expose-services=, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.buildah.version=1.24.2, vendor=Red Hat, Inc., distribution-scope=public, maintainer=Paul Cuzner , build-date=2022-03-28T10:36:18.413762, name=ubi8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=236.1648460182, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-04-02T04:59:09.178 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:59:08 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-02T04:59:09.178 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:59:08 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-02T04:59:10.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:10 smithi086 ceph-mon[24811]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:59:10.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:10 smithi086 ceph-mon[29505]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:59:10.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:10 smithi192 ceph-mon[25697]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:59:11.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:11 smithi086 ceph-mon[24811]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:59:11.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:11 smithi086 ceph-mon[29505]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:59:12.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:11 smithi192 ceph-mon[25697]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:59:13.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:12 smithi086 ceph-mon[24811]: from='client.15132 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:59:13.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:12 smithi086 ceph-mon[29505]: from='client.15132 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:59:13.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:12 smithi192 ceph-mon[25697]: from='client.15132 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:59:14.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:13 smithi086 ceph-mon[24811]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:59:14.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:13 smithi086 ceph-mon[29505]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:59:14.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:13 smithi192 ceph-mon[25697]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:59:16.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:16 smithi086 ceph-mon[24811]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:59:16.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:16 smithi086 ceph-mon[29505]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:59:16.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:16 smithi192 ceph-mon[25697]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:59:17.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:59:17] "GET /metrics HTTP/1.1" 200 37965 "" "Prometheus/2.33.4" 2024-04-02T04:59:18.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:18 smithi086 ceph-mon[24811]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:59:18.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:18 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:59:18.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:18 smithi086 ceph-mon[29505]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:59:18.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:18 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:59:18.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:18 smithi192 ceph-mon[25697]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:59:18.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:18 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:59:18.928 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:59:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T04:59:18.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"smithi192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"172.21.15.192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-02T04:59:19.428 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:59:18 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@grafana.a.service: Scheduled restart job, restart counter is at 5. 2024-04-02T04:59:19.428 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:59:18 smithi192 systemd[1]: Stopped Ceph grafana.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T04:59:19.428 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:59:18 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@grafana.a.service: Start request repeated too quickly. 2024-04-02T04:59:19.428 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:59:18 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-02T04:59:19.429 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 04:59:18 smithi192 systemd[1]: Failed to start Ceph grafana.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T04:59:20.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:20 smithi086 ceph-mon[24811]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:59:20.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:20 smithi086 ceph-mon[29505]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:59:20.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:20 smithi192 ceph-mon[25697]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:59:21.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:21 smithi086 ceph-mon[29505]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:59:21.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:21 smithi086 ceph-mon[24811]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:59:22.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:21 smithi192 ceph-mon[25697]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:59:23.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:22 smithi086 ceph-mon[29505]: from='client.15132 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:59:23.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:22 smithi086 ceph-mon[24811]: from='client.15132 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:59:23.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:22 smithi192 ceph-mon[25697]: from='client.15132 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:59:24.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:23 smithi086 ceph-mon[29505]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:59:24.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:23 smithi086 ceph-mon[24811]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:59:24.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:23 smithi192 ceph-mon[25697]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:59:26.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:26 smithi086 ceph-mon[29505]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:59:26.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:26 smithi086 ceph-mon[24811]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:59:26.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:26 smithi192 ceph-mon[25697]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:59:27.580 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:27 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:59:27] "GET /metrics HTTP/1.1" 200 37964 "" "Prometheus/2.33.4" 2024-04-02T04:59:28.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:28 smithi086 ceph-mon[24811]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:59:28.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:28 smithi086 ceph-mon[29505]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:59:28.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:28 smithi192 ceph-mon[25697]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:59:28.928 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:59:28 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T04:59:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"smithi192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"172.21.15.192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-02T04:59:30.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:30 smithi086 ceph-mon[29505]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:59:30.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:30 smithi086 ceph-mon[24811]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:59:30.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:30 smithi192 ceph-mon[25697]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T04:59:31.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:31 smithi086 ceph-mon[24811]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:59:31.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:31 smithi086 ceph-mon[29505]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:59:32.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:31 smithi192 ceph-mon[25697]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:59:32.864 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:32 smithi192 ceph-mon[25697]: from='client.15132 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:59:33.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:32 smithi086 ceph-mon[29505]: from='client.15132 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:59:33.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:32 smithi086 ceph-mon[24811]: from='client.15132 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T04:59:34.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:33 smithi086 ceph-mon[29505]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:59:34.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:33 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:59:34.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:33 smithi086 ceph-mon[24811]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:59:34.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:33 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:59:34.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:33 smithi192 ceph-mon[25697]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T04:59:34.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:33 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:59:35.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:34 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:59:35.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:34 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:59:35.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:34 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:59:35.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:34 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:59:35.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:34 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:59:35.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:34 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:59:35.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:34 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:59:35.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:34 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T04:59:35.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:34 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T04:59:35.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:34 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:59:35.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:34 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:59:35.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:34 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:59:35.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:34 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:59:35.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:34 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:59:35.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:34 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:59:35.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:34 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:59:35.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:34 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:59:35.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:34 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T04:59:35.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:34 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T04:59:35.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:34 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:59:35.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:34 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:59:35.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:34 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:59:35.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:34 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:59:35.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:34 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:59:35.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:34 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:59:35.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:34 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:59:35.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:34 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T04:59:35.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:34 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T04:59:35.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:34 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T04:59:35.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:34 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:59:36.382 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:36 smithi086 ceph-mon[29505]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-02T04:59:36.382 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:36 smithi086 ceph-mon[29505]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T04:59:36.382 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:36 smithi086 ceph-mon[29505]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T04:59:36.382 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:36 smithi086 ceph-mon[29505]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T04:59:36.383 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:36 smithi086 ceph-mon[24811]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-02T04:59:36.383 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:36 smithi086 ceph-mon[24811]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T04:59:36.383 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:36 smithi086 ceph-mon[24811]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T04:59:36.383 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:36 smithi086 ceph-mon[24811]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T04:59:36.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:36 smithi192 ceph-mon[25697]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-02T04:59:36.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:36 smithi192 ceph-mon[25697]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T04:59:36.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:36 smithi192 ceph-mon[25697]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T04:59:36.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:36 smithi192 ceph-mon[25697]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T04:59:37.428 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:59:37] "GET /metrics HTTP/1.1" 200 37968 "" "Prometheus/2.33.4" 2024-04-02T04:59:37.929 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T04:59:37.888+0000 7f9e1a22d700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T04:59:37.929 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Creating ceph-iscsi config... 2024-04-02T04:59:37.929 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T04:59:37.929 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T04:59:37.929 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T04:59:37.929 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:59:37.929 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:59:37.929 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Traceback (most recent call last): 2024-04-02T04:59:37.930 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T04:59:37.930 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: return _run_code(code, main_globals, None, 2024-04-02T04:59:37.930 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T04:59:37.930 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: exec(code, run_globals) 2024-04-02T04:59:37.930 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T04:59:37.930 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T04:59:37.930 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T04:59:37.930 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T04:59:37.930 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T04:59:37.930 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T04:59:37.930 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T04:59:37.930 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:59:37.930 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:59:37.931 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Traceback (most recent call last): 2024-04-02T04:59:37.931 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T04:59:37.931 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T04:59:37.931 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T04:59:37.931 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T04:59:37.931 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T04:59:37.931 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: return self.event_loop.get_result(coro, timeout) 2024-04-02T04:59:37.931 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T04:59:37.931 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: return future.result(timeout) 2024-04-02T04:59:37.931 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T04:59:37.931 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: return self.__get_result() 2024-04-02T04:59:37.931 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T04:59:37.931 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: raise self._exception 2024-04-02T04:59:37.931 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T04:59:37.932 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: config_blobs=daemon_spec.final_config, 2024-04-02T04:59:37.932 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T04:59:37.932 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T04:59:37.932 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T04:59:37.932 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Creating ceph-iscsi config... 2024-04-02T04:59:37.932 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T04:59:37.932 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T04:59:37.932 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T04:59:37.932 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:59:37.933 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:59:37.933 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Traceback (most recent call last): 2024-04-02T04:59:37.933 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T04:59:37.933 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: return _run_code(code, main_globals, None, 2024-04-02T04:59:37.933 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T04:59:37.933 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: exec(code, run_globals) 2024-04-02T04:59:37.933 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T04:59:37.933 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T04:59:37.933 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T04:59:37.934 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T04:59:37.934 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T04:59:37.934 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T04:59:37.934 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T04:59:37.934 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:59:37.934 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:59:38.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:38 smithi086 ceph-mon[29505]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T04:59:38.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:38 smithi086 ceph-mon[24811]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T04:59:38.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:38 smithi192 ceph-mon[25697]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T04:59:38.928 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:59:38 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T04:59:38.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"smithi192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"172.21.15.192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-02T04:59:39.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:39 smithi192 ceph-mon[25697]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T04:59:39.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: Creating ceph-iscsi config... 2024-04-02T04:59:39.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T04:59:39.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T04:59:39.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T04:59:39.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:59:39.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:59:39.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: Traceback (most recent call last): 2024-04-02T04:59:39.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T04:59:39.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: return _run_code(code, main_globals, None, 2024-04-02T04:59:39.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T04:59:39.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: exec(code, run_globals) 2024-04-02T04:59:39.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T04:59:39.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T04:59:39.432 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T04:59:39.432 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T04:59:39.432 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T04:59:39.432 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T04:59:39.432 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T04:59:39.432 INFO:journalctl@ceph.mon.b.smithi192.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:59:39.432 INFO:journalctl@ceph.mon.b.smithi192.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:59:39.432 INFO:journalctl@ceph.mon.b.smithi192.stdout: Traceback (most recent call last): 2024-04-02T04:59:39.432 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T04:59:39.432 INFO:journalctl@ceph.mon.b.smithi192.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T04:59:39.432 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T04:59:39.432 INFO:journalctl@ceph.mon.b.smithi192.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T04:59:39.433 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T04:59:39.433 INFO:journalctl@ceph.mon.b.smithi192.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-02T04:59:39.433 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T04:59:39.433 INFO:journalctl@ceph.mon.b.smithi192.stdout: return future.result(timeout) 2024-04-02T04:59:39.433 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T04:59:39.433 INFO:journalctl@ceph.mon.b.smithi192.stdout: return self.__get_result() 2024-04-02T04:59:39.433 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T04:59:39.433 INFO:journalctl@ceph.mon.b.smithi192.stdout: raise self._exception 2024-04-02T04:59:39.433 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T04:59:39.433 INFO:journalctl@ceph.mon.b.smithi192.stdout: config_blobs=daemon_spec.final_config, 2024-04-02T04:59:39.433 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T04:59:39.433 INFO:journalctl@ceph.mon.b.smithi192.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T04:59:39.433 INFO:journalctl@ceph.mon.b.smithi192.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T04:59:39.434 INFO:journalctl@ceph.mon.b.smithi192.stdout: Creating ceph-iscsi config... 2024-04-02T04:59:39.434 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T04:59:39.434 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T04:59:39.434 INFO:journalctl@ceph.mon.b.smithi192.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T04:59:39.434 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:59:39.434 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:59:39.434 INFO:journalctl@ceph.mon.b.smithi192.stdout: Traceback (most recent call last): 2024-04-02T04:59:39.434 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T04:59:39.434 INFO:journalctl@ceph.mon.b.smithi192.stdout: return _run_code(code, main_globals, None, 2024-04-02T04:59:39.434 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T04:59:39.434 INFO:journalctl@ceph.mon.b.smithi192.stdout: exec(code, run_globals) 2024-04-02T04:59:39.434 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T04:59:39.434 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T04:59:39.435 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T04:59:39.435 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T04:59:39.435 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T04:59:39.435 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T04:59:39.435 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T04:59:39.435 INFO:journalctl@ceph.mon.b.smithi192.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:59:39.435 INFO:journalctl@ceph.mon.b.smithi192.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:59:39.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:39 smithi086 ceph-mon[24811]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T04:59:39.586 INFO:journalctl@ceph.mon.a.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T04:59:39.586 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T04:59:39.586 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T04:59:39.586 INFO:journalctl@ceph.mon.a.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T04:59:39.586 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:59:39.586 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:59:39.586 INFO:journalctl@ceph.mon.a.smithi086.stdout: Traceback (most recent call last): 2024-04-02T04:59:39.586 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T04:59:39.586 INFO:journalctl@ceph.mon.a.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T04:59:39.586 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T04:59:39.586 INFO:journalctl@ceph.mon.a.smithi086.stdout: exec(code, run_globals) 2024-04-02T04:59:39.586 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T04:59:39.587 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T04:59:39.587 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T04:59:39.587 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T04:59:39.587 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T04:59:39.587 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T04:59:39.587 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T04:59:39.587 INFO:journalctl@ceph.mon.a.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:59:39.587 INFO:journalctl@ceph.mon.a.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:59:39.587 INFO:journalctl@ceph.mon.a.smithi086.stdout: Traceback (most recent call last): 2024-04-02T04:59:39.587 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T04:59:39.587 INFO:journalctl@ceph.mon.a.smithi086.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T04:59:39.587 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T04:59:39.587 INFO:journalctl@ceph.mon.a.smithi086.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T04:59:39.587 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T04:59:39.588 INFO:journalctl@ceph.mon.a.smithi086.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-02T04:59:39.588 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T04:59:39.588 INFO:journalctl@ceph.mon.a.smithi086.stdout: return future.result(timeout) 2024-04-02T04:59:39.588 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T04:59:39.588 INFO:journalctl@ceph.mon.a.smithi086.stdout: return self.__get_result() 2024-04-02T04:59:39.588 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T04:59:39.588 INFO:journalctl@ceph.mon.a.smithi086.stdout: raise self._exception 2024-04-02T04:59:39.588 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T04:59:39.588 INFO:journalctl@ceph.mon.a.smithi086.stdout: config_blobs=daemon_spec.final_config, 2024-04-02T04:59:39.588 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T04:59:39.588 INFO:journalctl@ceph.mon.a.smithi086.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T04:59:39.588 INFO:journalctl@ceph.mon.a.smithi086.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T04:59:39.588 INFO:journalctl@ceph.mon.a.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T04:59:39.588 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T04:59:39.588 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T04:59:39.589 INFO:journalctl@ceph.mon.a.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T04:59:39.589 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:59:39.589 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:59:39.589 INFO:journalctl@ceph.mon.a.smithi086.stdout: Traceback (most recent call last): 2024-04-02T04:59:39.589 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T04:59:39.589 INFO:journalctl@ceph.mon.a.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T04:59:39.589 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T04:59:39.589 INFO:journalctl@ceph.mon.a.smithi086.stdout: exec(code, run_globals) 2024-04-02T04:59:39.589 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T04:59:39.589 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T04:59:39.589 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T04:59:39.590 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T04:59:39.590 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T04:59:39.590 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T04:59:39.590 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T04:59:39.590 INFO:journalctl@ceph.mon.a.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:59:39.590 INFO:journalctl@ceph.mon.a.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:59:39.590 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:39 smithi086 ceph-mon[29505]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T04:59:39.590 INFO:journalctl@ceph.mon.c.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T04:59:39.591 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T04:59:39.591 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T04:59:39.591 INFO:journalctl@ceph.mon.c.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T04:59:39.591 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:59:39.591 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:59:39.591 INFO:journalctl@ceph.mon.c.smithi086.stdout: Traceback (most recent call last): 2024-04-02T04:59:39.591 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T04:59:39.591 INFO:journalctl@ceph.mon.c.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T04:59:39.591 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T04:59:39.591 INFO:journalctl@ceph.mon.c.smithi086.stdout: exec(code, run_globals) 2024-04-02T04:59:39.591 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T04:59:39.591 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T04:59:39.591 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T04:59:39.591 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T04:59:39.592 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T04:59:39.592 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T04:59:39.592 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T04:59:39.592 INFO:journalctl@ceph.mon.c.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:59:39.592 INFO:journalctl@ceph.mon.c.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:59:39.592 INFO:journalctl@ceph.mon.c.smithi086.stdout: Traceback (most recent call last): 2024-04-02T04:59:39.592 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T04:59:39.592 INFO:journalctl@ceph.mon.c.smithi086.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T04:59:39.592 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T04:59:39.592 INFO:journalctl@ceph.mon.c.smithi086.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T04:59:39.592 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T04:59:39.592 INFO:journalctl@ceph.mon.c.smithi086.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-02T04:59:39.592 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T04:59:39.592 INFO:journalctl@ceph.mon.c.smithi086.stdout: return future.result(timeout) 2024-04-02T04:59:39.593 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T04:59:39.593 INFO:journalctl@ceph.mon.c.smithi086.stdout: return self.__get_result() 2024-04-02T04:59:39.593 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T04:59:39.593 INFO:journalctl@ceph.mon.c.smithi086.stdout: raise self._exception 2024-04-02T04:59:39.593 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T04:59:39.593 INFO:journalctl@ceph.mon.c.smithi086.stdout: config_blobs=daemon_spec.final_config, 2024-04-02T04:59:39.593 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T04:59:39.593 INFO:journalctl@ceph.mon.c.smithi086.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T04:59:39.593 INFO:journalctl@ceph.mon.c.smithi086.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T04:59:39.593 INFO:journalctl@ceph.mon.c.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T04:59:39.593 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T04:59:39.593 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T04:59:39.593 INFO:journalctl@ceph.mon.c.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T04:59:39.593 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:59:39.594 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:59:39.594 INFO:journalctl@ceph.mon.c.smithi086.stdout: Traceback (most recent call last): 2024-04-02T04:59:39.594 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T04:59:39.594 INFO:journalctl@ceph.mon.c.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T04:59:39.594 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T04:59:39.594 INFO:journalctl@ceph.mon.c.smithi086.stdout: exec(code, run_globals) 2024-04-02T04:59:39.594 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T04:59:39.594 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T04:59:39.594 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T04:59:39.594 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T04:59:39.594 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T04:59:39.594 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T04:59:39.595 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T04:59:39.595 INFO:journalctl@ceph.mon.c.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T04:59:39.595 INFO:journalctl@ceph.mon.c.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T04:59:40.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:40 smithi192 ceph-mon[25697]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 525 B/s rd, 0 op/s 2024-04-02T04:59:40.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:40 smithi086 ceph-mon[24811]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 525 B/s rd, 0 op/s 2024-04-02T04:59:40.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:40 smithi086 ceph-mon[29505]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 525 B/s rd, 0 op/s 2024-04-02T04:59:42.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:42 smithi192 ceph-mon[25697]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 840 B/s rd, 0 op/s 2024-04-02T04:59:42.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:42 smithi086 ceph-mon[24811]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 840 B/s rd, 0 op/s 2024-04-02T04:59:42.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:42 smithi086 ceph-mon[29505]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 840 B/s rd, 0 op/s 2024-04-02T04:59:44.386 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:44 smithi086 ceph-mon[29505]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 840 B/s rd, 0 op/s 2024-04-02T04:59:44.386 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:44 smithi086 ceph-mon[24811]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 840 B/s rd, 0 op/s 2024-04-02T04:59:44.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:44 smithi192 ceph-mon[25697]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 840 B/s rd, 0 op/s 2024-04-02T04:59:46.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:46 smithi192 ceph-mon[25697]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 307 B/s rd, 0 op/s 2024-04-02T04:59:46.541 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:46 smithi086 ceph-mon[29505]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 307 B/s rd, 0 op/s 2024-04-02T04:59:46.541 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:46 smithi086 ceph-mon[24811]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 307 B/s rd, 0 op/s 2024-04-02T04:59:47.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:59:47] "GET /metrics HTTP/1.1" 200 37968 "" "Prometheus/2.33.4" 2024-04-02T04:59:48.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:48 smithi192 ceph-mon[25697]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-02T04:59:48.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:48 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:59:48.476 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:48 smithi086 ceph-mon[24811]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-02T04:59:48.477 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:48 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:59:48.477 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:48 smithi086 ceph-mon[29505]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-02T04:59:48.477 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:48 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:59:48.928 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:59:48 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T04:59:48.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"smithi192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"172.21.15.192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-02T04:59:50.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:50 smithi192 ceph-mon[25697]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-02T04:59:50.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:50 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/548411439' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T04:59:50.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:50 smithi086 ceph-mon[24811]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-02T04:59:50.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:50 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/548411439' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T04:59:50.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:50 smithi086 ceph-mon[29505]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-02T04:59:50.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:50 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/548411439' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T04:59:52.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:52 smithi192 ceph-mon[25697]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-02T04:59:52.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:52 smithi086 ceph-mon[24811]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-02T04:59:52.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:52 smithi086 ceph-mon[29505]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-02T04:59:54.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:54 smithi192 ceph-mon[25697]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:59:54.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:54 smithi086 ceph-mon[24811]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:59:54.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:54 smithi086 ceph-mon[29505]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail 2024-04-02T04:59:56.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:56 smithi192 ceph-mon[25697]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-02T04:59:56.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:56 smithi086 ceph-mon[24811]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-02T04:59:56.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:56 smithi086 ceph-mon[29505]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-02T04:59:57.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 04:59:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:04:59:57] "GET /metrics HTTP/1.1" 200 37980 "" "Prometheus/2.33.4" 2024-04-02T04:59:58.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 04:59:58 smithi192 ceph-mon[25697]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-02T04:59:58.527 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 04:59:58 smithi086 ceph-mon[29505]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-02T04:59:58.527 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 04:59:58 smithi086 ceph-mon[24811]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-02T04:59:58.928 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 04:59:58 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T04:59:58.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"smithi192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"172.21.15.192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-02T05:00:00.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:00 smithi192 ceph-mon[25697]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-02T05:00:00.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:00 smithi192 ceph-mon[25697]: from='client.25051 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:00:00.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:00 smithi192 ceph-mon[25697]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-02T05:00:00.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:00 smithi192 ceph-mon[25697]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-02T05:00:00.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:00 smithi192 ceph-mon[25697]: daemon grafana.a on smithi192 is in error state 2024-04-02T05:00:00.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:00 smithi086 ceph-mon[29505]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-02T05:00:00.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:00 smithi086 ceph-mon[29505]: from='client.25051 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:00:00.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:00 smithi086 ceph-mon[29505]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-02T05:00:00.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:00 smithi086 ceph-mon[29505]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-02T05:00:00.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:00 smithi086 ceph-mon[29505]: daemon grafana.a on smithi192 is in error state 2024-04-02T05:00:00.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:00 smithi086 ceph-mon[24811]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-02T05:00:00.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:00 smithi086 ceph-mon[24811]: from='client.25051 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:00:00.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:00 smithi086 ceph-mon[24811]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-02T05:00:00.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:00 smithi086 ceph-mon[24811]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-02T05:00:00.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:00 smithi086 ceph-mon[24811]: daemon grafana.a on smithi192 is in error state 2024-04-02T05:00:02.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:02 smithi192 ceph-mon[25697]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:02.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:02 smithi086 ceph-mon[24811]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:02.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:02 smithi086 ceph-mon[29505]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:03.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:03 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:00:03.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:03 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:00:03.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:03 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:00:04.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:04 smithi192 ceph-mon[25697]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:04.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:04 smithi086 ceph-mon[24811]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:04.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:04 smithi086 ceph-mon[29505]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:06.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:06 smithi192 ceph-mon[25697]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:00:06.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:06 smithi086 ceph-mon[24811]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:00:06.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:06 smithi086 ceph-mon[29505]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:00:07.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:00:07 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:00:07] "GET /metrics HTTP/1.1" 200 37978 "" "Prometheus/2.33.4" 2024-04-02T05:00:08.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:08 smithi192 ceph-mon[25697]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:08.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:08 smithi086 ceph-mon[24811]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:08.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:08 smithi086 ceph-mon[29505]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:08.928 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:00:08 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T05:00:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"smithi192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"172.21.15.192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-02T05:00:10.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:10 smithi192 ceph-mon[25697]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:10.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:10 smithi192 ceph-mon[25697]: from='client.25051 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:00:10.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:10 smithi086 ceph-mon[24811]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:10.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:10 smithi086 ceph-mon[24811]: from='client.25051 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:00:10.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:10 smithi086 ceph-mon[29505]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:10.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:10 smithi086 ceph-mon[29505]: from='client.25051 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:00:12.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:12 smithi192 ceph-mon[25697]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:00:12.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:12 smithi086 ceph-mon[24811]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:00:12.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:12 smithi086 ceph-mon[29505]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:00:14.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:14 smithi192 ceph-mon[25697]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:14.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:14 smithi086 ceph-mon[24811]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:14.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:14 smithi086 ceph-mon[29505]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:16.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:16 smithi192 ceph-mon[25697]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:00:16.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:16 smithi086 ceph-mon[29505]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:00:16.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:16 smithi086 ceph-mon[24811]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:00:17.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:00:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:00:17] "GET /metrics HTTP/1.1" 200 37978 "" "Prometheus/2.33.4" 2024-04-02T05:00:18.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:18 smithi086 ceph-mon[24811]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:18.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:18 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:00:18.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:18 smithi086 ceph-mon[29505]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:18.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:18 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:00:18.663 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:18 smithi192 ceph-mon[25697]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:18.663 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:18 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:00:18.929 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:00:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T05:00:18.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"smithi192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"172.21.15.192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-02T05:00:20.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:20 smithi086 ceph-mon[24811]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:20.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:20 smithi086 ceph-mon[24811]: from='client.25051 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:00:20.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:20 smithi086 ceph-mon[29505]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:20.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:20 smithi086 ceph-mon[29505]: from='client.25051 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:00:20.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:20 smithi192 ceph-mon[25697]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:20.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:20 smithi192 ceph-mon[25697]: from='client.25051 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:00:22.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:22 smithi086 ceph-mon[29505]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:00:22.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:22 smithi086 ceph-mon[24811]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:00:22.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:22 smithi192 ceph-mon[25697]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:00:24.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:24 smithi086 ceph-mon[24811]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:24.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:24 smithi086 ceph-mon[29505]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:24.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:24 smithi192 ceph-mon[25697]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:26.480 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:26 smithi192 ceph-mon[25697]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:00:26.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:26 smithi086 ceph-mon[24811]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:00:26.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:26 smithi086 ceph-mon[29505]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:00:27.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:00:27 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:00:27] "GET /metrics HTTP/1.1" 200 37974 "" "Prometheus/2.33.4" 2024-04-02T05:00:28.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:28 smithi086 ceph-mon[29505]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:28.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:28 smithi086 ceph-mon[24811]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:28.662 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:28 smithi192 ceph-mon[25697]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:28.928 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:00:28 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T05:00:28.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"smithi192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"172.21.15.192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-02T05:00:30.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:30 smithi086 ceph-mon[24811]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:30.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:30 smithi086 ceph-mon[24811]: from='client.25051 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:00:30.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:30 smithi086 ceph-mon[29505]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:30.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:30 smithi086 ceph-mon[29505]: from='client.25051 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:00:30.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:30 smithi192 ceph-mon[25697]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:30.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:30 smithi192 ceph-mon[25697]: from='client.25051 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:00:32.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:32 smithi086 ceph-mon[24811]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:00:32.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:32 smithi086 ceph-mon[29505]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:00:32.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:32 smithi192 ceph-mon[25697]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:00:33.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:33 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:00:33.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:33 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:00:33.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:33 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:00:34.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:34 smithi086 ceph-mon[24811]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:34.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:34 smithi086 ceph-mon[29505]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:34.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:34 smithi192 ceph-mon[25697]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:35.927 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-02T05:00:36.466 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:36 smithi086 ceph-mon[24811]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:00:36.466 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:36 smithi086 ceph-mon[29505]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:00:36.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:36 smithi192 ceph-mon[25697]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:00:37.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:00:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:00:37] "GET /metrics HTTP/1.1" 200 37976 "" "Prometheus/2.33.4" 2024-04-02T05:00:38.213 INFO:teuthology.orchestra.run.smithi086.stdout:Scheduled to redeploy mgr.y on host 'smithi086' 2024-04-02T05:00:38.483 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:38 smithi086 ceph-mon[29505]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:38.484 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:38 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/2730443267' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-02T05:00:38.484 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:38 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:00:38.484 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:38 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:00:38.484 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:38 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:00:38.484 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:38 smithi086 ceph-mon[24811]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:38.484 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:38 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/2730443267' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-02T05:00:38.484 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:38 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:00:38.484 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:38 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:00:38.484 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:38 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:00:38.662 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:38 smithi192 ceph-mon[25697]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:38.663 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:38 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/2730443267' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-02T05:00:38.663 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:38 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:00:38.663 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:38 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:00:38.663 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:38 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:00:38.825 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch ps --refresh' 2024-04-02T05:00:38.928 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:00:38 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T05:00:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"smithi192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"172.21.15.192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-02T05:00:39.402 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:39 smithi086 ceph-mon[29505]: from='client.15177 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:00:39.402 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:39 smithi086 ceph-mon[29505]: Schedule redeploy daemon mgr.y 2024-04-02T05:00:39.402 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:39 smithi086 ceph-mon[24811]: from='client.15177 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:00:39.402 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:39 smithi086 ceph-mon[24811]: Schedule redeploy daemon mgr.y 2024-04-02T05:00:39.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:39 smithi192 ceph-mon[25697]: from='client.15177 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:00:39.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:39 smithi192 ceph-mon[25697]: Schedule redeploy daemon mgr.y 2024-04-02T05:00:40.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:40 smithi192 ceph-mon[25697]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:40.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:40 smithi192 ceph-mon[25697]: from='client.25051 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:00:40.460 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:40 smithi086 ceph-mon[24811]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:40.461 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:40 smithi086 ceph-mon[24811]: from='client.25051 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:00:40.461 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:40 smithi086 ceph-mon[29505]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:00:40.461 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:40 smithi086 ceph-mon[29505]: from='client.25051 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:00:40.777 INFO:teuthology.orchestra.run.smithi086.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T05:00:40.777 INFO:teuthology.orchestra.run.smithi086.stdout:alertmanager.a smithi086 *:9093,9094 running (2m) 66s ago 10m 16.9M - 0.23.0 ba2b418f427c b2769c8e287a 2024-04-02T05:00:40.777 INFO:teuthology.orchestra.run.smithi086.stdout:grafana.a smithi192 *:3000 error 66s ago 10m - - 2024-04-02T05:00:40.777 INFO:teuthology.orchestra.run.smithi086.stdout:iscsi.foo.smithi086.rsbqwx smithi086 running (119s) 66s ago 9m 46.2M - 3.5 e1d6a67b021e 53b73149f319 2024-04-02T05:00:40.777 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.x smithi192 *:8443,9283 running (5m) 66s ago 15m 508M - 19.0.0-1979-g6cec793d 78f830190839 fd618149d6b4 2024-04-02T05:00:40.777 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.y smithi086 *:9283 running (16m) 66s ago 16m 411M - 17.2.0 e1d6a67b021e 02f336a19fd6 2024-04-02T05:00:40.777 INFO:teuthology.orchestra.run.smithi086.stdout:mon.a smithi086 running (16m) 66s ago 16m 55.2M 2048M 17.2.0 e1d6a67b021e b6c507ab96f8 2024-04-02T05:00:40.777 INFO:teuthology.orchestra.run.smithi086.stdout:mon.b smithi192 running (15m) 66s ago 15m 48.5M 2048M 17.2.0 e1d6a67b021e 4dadd8d1c318 2024-04-02T05:00:40.777 INFO:teuthology.orchestra.run.smithi086.stdout:mon.c smithi086 running (15m) 66s ago 15m 43.8M 2048M 17.2.0 e1d6a67b021e 49275483a2d4 2024-04-02T05:00:40.778 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.a smithi086 *:9100 running (2m) 66s ago 11m 16.5M - 1.3.1 1dbe0e931976 843a5e9887fb 2024-04-02T05:00:40.778 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.b smithi192 *:9100 running (2m) 66s ago 11m 8334k - 1.3.1 1dbe0e931976 bd849d6c4fb0 2024-04-02T05:00:40.778 INFO:teuthology.orchestra.run.smithi086.stdout:osd.0 smithi086 running (14m) 66s ago 14m 54.6M 2992M 17.2.0 e1d6a67b021e 6d90c60d1c71 2024-04-02T05:00:40.778 INFO:teuthology.orchestra.run.smithi086.stdout:osd.1 smithi086 running (14m) 66s ago 14m 53.4M 2992M 17.2.0 e1d6a67b021e 203689f71603 2024-04-02T05:00:40.778 INFO:teuthology.orchestra.run.smithi086.stdout:osd.2 smithi086 running (13m) 66s ago 13m 48.5M 2992M 17.2.0 e1d6a67b021e 472c860a9af8 2024-04-02T05:00:40.778 INFO:teuthology.orchestra.run.smithi086.stdout:osd.3 smithi086 running (13m) 66s ago 13m 50.4M 2992M 17.2.0 e1d6a67b021e 327538dac44e 2024-04-02T05:00:40.778 INFO:teuthology.orchestra.run.smithi086.stdout:osd.4 smithi192 running (13m) 66s ago 13m 51.9M 3248M 17.2.0 e1d6a67b021e 6535d7919a2d 2024-04-02T05:00:40.778 INFO:teuthology.orchestra.run.smithi086.stdout:osd.5 smithi192 running (12m) 66s ago 12m 51.9M 3248M 17.2.0 e1d6a67b021e 7259249605f2 2024-04-02T05:00:40.778 INFO:teuthology.orchestra.run.smithi086.stdout:osd.6 smithi192 running (12m) 66s ago 12m 49.6M 3248M 17.2.0 e1d6a67b021e bb7d171b02a1 2024-04-02T05:00:40.778 INFO:teuthology.orchestra.run.smithi086.stdout:osd.7 smithi192 running (12m) 66s ago 12m 51.8M 3248M 17.2.0 e1d6a67b021e 95f5a0353999 2024-04-02T05:00:40.778 INFO:teuthology.orchestra.run.smithi086.stdout:prometheus.a smithi192 *:9095 running (2m) 66s ago 11m 53.9M - 2.33.4 514e6a882f6e d3ec96f3bde3 2024-04-02T05:00:40.779 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi086.bxzxlp smithi086 *:8000 running (9m) 66s ago 9m 74.4M - 17.2.0 e1d6a67b021e 6725785fbc53 2024-04-02T05:00:40.779 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi192.pmlbot smithi192 *:8000 running (9m) 66s ago 9m 73.3M - 17.2.0 e1d6a67b021e 5ae3638f8b1c 2024-04-02T05:00:41.354 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'sleep 180' 2024-04-02T05:00:42.580 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:42 smithi086 ceph-mon[29505]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:00:42.580 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:42 smithi086 ceph-mon[29505]: from='client.15183 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:00:42.580 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:42 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:00:42.580 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:42 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:00:42.580 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:42 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:00:42.580 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:42 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:00:42.580 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:42 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:00:42.580 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:42 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T05:00:42.580 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:42 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T05:00:42.581 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:42 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T05:00:42.581 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:42 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:00:42.581 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:42 smithi086 ceph-mon[24811]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:00:42.581 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:42 smithi086 ceph-mon[24811]: from='client.15183 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:00:42.581 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:42 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:00:42.581 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:42 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:00:42.581 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:42 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:00:42.581 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:42 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:00:42.581 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:42 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:00:42.581 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:42 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T05:00:42.581 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:42 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T05:00:42.582 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:42 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T05:00:42.582 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:42 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:00:42.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:42 smithi192 ceph-mon[25697]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:00:42.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:42 smithi192 ceph-mon[25697]: from='client.15183 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:00:42.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:42 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:00:42.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:42 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:00:42.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:42 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:00:42.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:42 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:00:42.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:42 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:00:42.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:42 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T05:00:42.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:42 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T05:00:42.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:42 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T05:00:42.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:42 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:00:43.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:43 smithi086 ceph-mon[24811]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 900 B/s rd, 0 op/s 2024-04-02T05:00:43.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:43 smithi086 ceph-mon[24811]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:00:43.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:43 smithi086 ceph-mon[24811]: Deploying daemon mgr.y on smithi086 2024-04-02T05:00:43.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:43 smithi086 ceph-mon[29505]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 900 B/s rd, 0 op/s 2024-04-02T05:00:43.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:43 smithi086 ceph-mon[29505]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:00:43.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:43 smithi086 ceph-mon[29505]: Deploying daemon mgr.y on smithi086 2024-04-02T05:00:43.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:43 smithi192 ceph-mon[25697]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 900 B/s rd, 0 op/s 2024-04-02T05:00:43.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:43 smithi192 ceph-mon[25697]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:00:43.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:43 smithi192 ceph-mon[25697]: Deploying daemon mgr.y on smithi086 2024-04-02T05:00:45.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:45 smithi086 ceph-mon[24811]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 6.2 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-02T05:00:45.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:45 smithi086 ceph-mon[29505]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 6.2 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-02T05:00:45.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:45 smithi192 ceph-mon[25697]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 6.2 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-02T05:00:46.793 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:46 smithi086 ceph-mon[24811]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 6.2 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-02T05:00:47.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:46 smithi086 ceph-mon[29505]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 6.2 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-02T05:00:47.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:46 smithi192 ceph-mon[25697]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 6.2 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-02T05:00:47.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:00:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:00:47] "GET /metrics HTTP/1.1" 200 37976 "" "Prometheus/2.33.4" 2024-04-02T05:00:48.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:48 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:00:48.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:48 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:00:48.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:48 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:00:48.928 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:00:48 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T05:00:48.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"smithi192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"172.21.15.192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-02T05:00:49.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:49 smithi086 ceph-mon[24811]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 6.2 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-02T05:00:49.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:49 smithi086 ceph-mon[29505]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 6.2 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-02T05:00:49.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:49 smithi192 ceph-mon[25697]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 6.2 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-02T05:00:50.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:50 smithi086 ceph-mon[24811]: from='client.25051 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:00:50.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:50 smithi086 ceph-mon[29505]: from='client.25051 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:00:50.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:50 smithi192 ceph-mon[25697]: from='client.25051 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:00:51.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:51 smithi086 ceph-mon[29505]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s rd, 0 B/s wr, 153 op/s 2024-04-02T05:00:51.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:51 smithi086 ceph-mon[24811]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s rd, 0 B/s wr, 153 op/s 2024-04-02T05:00:51.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:51 smithi192 ceph-mon[25697]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s rd, 0 B/s wr, 153 op/s 2024-04-02T05:00:53.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:53 smithi192 ceph-mon[25697]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s rd, 0 B/s wr, 143 op/s 2024-04-02T05:00:53.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:53 smithi086 ceph-mon[29505]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s rd, 0 B/s wr, 143 op/s 2024-04-02T05:00:53.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:53 smithi086 ceph-mon[24811]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s rd, 0 B/s wr, 143 op/s 2024-04-02T05:00:55.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:55 smithi192 ceph-mon[25697]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-02T05:00:55.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:55 smithi086 ceph-mon[24811]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-02T05:00:55.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:55 smithi086 ceph-mon[29505]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-02T05:00:57.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:56 smithi086 ceph-mon[29505]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s rd, 0 B/s wr, 113 op/s 2024-04-02T05:00:57.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:56 smithi086 ceph-mon[24811]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s rd, 0 B/s wr, 113 op/s 2024-04-02T05:00:57.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:56 smithi192 ceph-mon[25697]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s rd, 0 B/s wr, 113 op/s 2024-04-02T05:00:57.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:00:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:00:57] "GET /metrics HTTP/1.1" 200 37967 "" "Prometheus/2.33.4" 2024-04-02T05:00:58.928 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:00:58 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T05:00:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"smithi192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"172.21.15.192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-02T05:00:59.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:00:59 smithi086 ceph-mon[24811]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s rd, 0 B/s wr, 113 op/s 2024-04-02T05:00:59.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:00:59 smithi086 ceph-mon[29505]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s rd, 0 B/s wr, 113 op/s 2024-04-02T05:00:59.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:00:59 smithi192 ceph-mon[25697]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s rd, 0 B/s wr, 113 op/s 2024-04-02T05:01:00.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:00 smithi192 ceph-mon[25697]: from='client.25051 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:01:00.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:00 smithi086 ceph-mon[24811]: from='client.25051 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:01:00.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:00 smithi086 ceph-mon[29505]: from='client.25051 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:01:01.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:01 smithi192 ceph-mon[25697]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 69 KiB/s rd, 0 B/s wr, 113 op/s 2024-04-02T05:01:01.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:01 smithi086 ceph-mon[24811]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 69 KiB/s rd, 0 B/s wr, 113 op/s 2024-04-02T05:01:01.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:01 smithi086 ceph-mon[29505]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 69 KiB/s rd, 0 B/s wr, 113 op/s 2024-04-02T05:01:03.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:03 smithi192 ceph-mon[25697]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:01:03.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:03 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:01:03.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:03 smithi086 ceph-mon[24811]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:01:03.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:03 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:01:03.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:03 smithi086 ceph-mon[29505]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:01:03.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:03 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:01:05.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:05 smithi192 ceph-mon[25697]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:01:05.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:05 smithi086 ceph-mon[24811]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:01:05.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:05 smithi086 ceph-mon[29505]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:01:07.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:06 smithi086 ceph-mon[24811]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:01:07.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:06 smithi086 ceph-mon[29505]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:01:07.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:06 smithi192 ceph-mon[25697]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:01:07.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:07 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:01:07] "GET /metrics HTTP/1.1" 200 37975 "" "Prometheus/2.33.4" 2024-04-02T05:01:08.928 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:01:08 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T05:01:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"smithi192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"172.21.15.192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-02T05:01:09.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:09 smithi086 ceph-mon[24811]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:01:09.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:09 smithi086 ceph-mon[29505]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:01:09.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:09 smithi192 ceph-mon[25697]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:01:10.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:10 smithi192 ceph-mon[25697]: from='client.25051 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:01:10.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:10 smithi086 ceph-mon[24811]: from='client.25051 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:01:10.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:10 smithi086 ceph-mon[29505]: from='client.25051 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:01:11.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:11 smithi086 ceph-mon[29505]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:01:11.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:11 smithi086 ceph-mon[24811]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:01:11.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:11 smithi192 ceph-mon[25697]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:01:13.278 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:13 smithi086 ceph-mon[29505]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:01:13.279 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:13 smithi086 ceph-mon[24811]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:01:13.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:13 smithi192 ceph-mon[25697]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:01:14.423 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:14 smithi086 systemd[1]: Stopping Ceph mgr.y for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:01:14.728 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:14 smithi086 podman[62482]: 2024-04-02 05:01:14.420344585 +0000 UTC m=+0.126391990 container stop 02f336a19fd69627a57257bbda89253ad8dfb6c357f5d7890e0b5b4083eeb81b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y, CEPH_POINT_RELEASE=-17.2.0, ceph=True, RELEASE=HEAD, io.openshift.expose-services=, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, release=754, architecture=x86_64, GIT_CLEAN=True, distribution-scope=public, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-02T05:01:14.728 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:14 smithi086 podman[62482]: 2024-04-02 05:01:14.451095882 +0000 UTC m=+0.157143321 container died 02f336a19fd69627a57257bbda89253ad8dfb6c357f5d7890e0b5b4083eeb81b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y, version=8, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, distribution-scope=public, vendor=Red Hat, Inc., vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-02T05:01:15.084 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:14 smithi086 podman[62482]: 2024-04-02 05:01:14.790769685 +0000 UTC m=+0.496817147 container cleanup 02f336a19fd69627a57257bbda89253ad8dfb6c357f5d7890e0b5b4083eeb81b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, io.openshift.expose-services=, architecture=x86_64, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, ceph=True, io.openshift.tags=base centos centos-stream, vcs-type=git, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8) 2024-04-02T05:01:15.084 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:14 smithi086 bash[62482]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y 2024-04-02T05:01:15.084 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:14 smithi086 podman[62497]: 2024-04-02 05:01:14.96706613 +0000 UTC m=+0.541593402 container remove 02f336a19fd69627a57257bbda89253ad8dfb6c357f5d7890e0b5b4083eeb81b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y, architecture=x86_64, io.buildah.version=1.19.8, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, io.k8s.display-name=CentOS Stream 8, version=8, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, vendor=Red Hat, Inc., release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, vcs-type=git, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , name=centos-stream, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public) 2024-04-02T05:01:15.084 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:14 smithi086 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-04-02T05:01:15.406 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:15 smithi086 ceph-mon[29505]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:01:15.407 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:15 smithi086 ceph-mon[24811]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:01:15.407 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:15 smithi086 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mgr.y.service: Failed with result 'exit-code'. 2024-04-02T05:01:15.407 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:15 smithi086 systemd[1]: Stopped Ceph mgr.y for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:01:15.407 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:15 smithi086 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mgr.y.service: Consumed 59.349s CPU time. 2024-04-02T05:01:15.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:15 smithi192 ceph-mon[25697]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:01:15.717 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:15 smithi086 systemd[1]: Starting Ceph mgr.y for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:01:15.717 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:15 smithi086 podman[62627]: 2024-04-02 05:01:15.593126522 +0000 UTC m=+0.090078599 container create 3dca54a018ab67df0860f4faa8227434aaeeb9e9d474a9a73c5988c7f830e60d (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y, ceph=True, RELEASE=squid-6cec793, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-04-02T05:01:15.717 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:15 smithi086 podman[62627]: 2024-04-02 05:01:15.520570885 +0000 UTC m=+0.017522963 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:01:16.041 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:15 smithi086 podman[62627]: 2024-04-02 05:01:15.812446274 +0000 UTC m=+0.309398352 container init 3dca54a018ab67df0860f4faa8227434aaeeb9e9d474a9a73c5988c7f830e60d (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux , RELEASE=squid-6cec793, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-04-02T05:01:16.042 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:15 smithi086 podman[62627]: 2024-04-02 05:01:15.815211583 +0000 UTC m=+0.312163662 container start 3dca54a018ab67df0860f4faa8227434aaeeb9e9d474a9a73c5988c7f830e60d (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-6cec793, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-04-02T05:01:16.042 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:15 smithi086 ceph-mgr[62645]: -- 172.21.15.86:0/1588390912 <== mon.0 v2:172.21.15.86:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56511bbee5a0 con 0x56511bbc2c00 2024-04-02T05:01:16.042 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:15 smithi086 bash[62627]: 3dca54a018ab67df0860f4faa8227434aaeeb9e9d474a9a73c5988c7f830e60d 2024-04-02T05:01:16.042 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:15 smithi086 systemd[1]: Started Ceph mgr.y for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:01:16.721 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:16 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:01:16.419+0000 7f4ad47f0200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-02T05:01:16.722 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:16 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:01:16.529+0000 7f4ad47f0200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-02T05:01:17.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:16 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:17.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:16 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:17.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:16 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:17.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:16 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:17.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:16 smithi086 ceph-mon[29505]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T05:01:17.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:16 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:01:17.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:16 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:01:17.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:16 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:01:17.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:16 smithi086 ceph-mon[29505]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T05:01:17.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:16 smithi086 ceph-mon[29505]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:01:17.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:16 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:17.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:16 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:17.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:16 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:17.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:16 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:17.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:16 smithi086 ceph-mon[24811]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T05:01:17.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:16 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:01:17.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:16 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:01:17.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:16 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:01:17.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:16 smithi086 ceph-mon[24811]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T05:01:17.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:16 smithi086 ceph-mon[24811]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:01:17.428 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:01:17] "GET /metrics HTTP/1.1" 200 37975 "" "Prometheus/2.33.4" 2024-04-02T05:01:17.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:16 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:17.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:16 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:17.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:16 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:17.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:16 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:17.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:16 smithi192 ceph-mon[25697]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T05:01:17.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:16 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:01:17.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:16 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:01:17.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:16 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:01:17.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:16 smithi192 ceph-mon[25697]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T05:01:17.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:16 smithi192 ceph-mon[25697]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:01:17.505 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:17 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:01:17.225+0000 7f4ad47f0200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-02T05:01:17.505 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:17 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:01:17.324+0000 7f4ad47f0200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-02T05:01:17.505 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:17 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:01:17.469+0000 7f4ad47f0200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-02T05:01:17.834 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:17 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:01:17.576+0000 7f4ad47f0200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-02T05:01:17.834 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:17 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:01:17.673+0000 7f4ad47f0200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-02T05:01:18.166 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:18 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:01:18.167 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:18 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:01:18.167 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:17 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:01:17.950+0000 7f4ad47f0200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-02T05:01:18.167 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:18 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:01:18.068+0000 7f4ad47f0200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-02T05:01:18.167 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:18 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:01:18.162+0000 7f4ad47f0200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-02T05:01:18.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:18 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:01:18.447 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:18 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:01:18.262+0000 7f4ad47f0200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-02T05:01:18.794 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:18 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:01:18.442+0000 7f4ad47f0200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-02T05:01:18.928 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:01:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T05:01:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"smithi192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"172.21.15.192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-02T05:01:19.054 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:19 smithi086 ceph-mon[24811]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:01:19.055 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:19 smithi086 ceph-mon[29505]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:01:19.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-mon[25697]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:01:19.428 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:01:19.144+0000 7f9e1a22d700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:01:19.428 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Creating ceph-iscsi config... 2024-04-02T05:01:19.428 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:01:19.428 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:01:19.428 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:01:19.428 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:01:19.429 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:01:19.429 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Traceback (most recent call last): 2024-04-02T05:01:19.429 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:01:19.429 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: return _run_code(code, main_globals, None, 2024-04-02T05:01:19.429 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:01:19.429 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: exec(code, run_globals) 2024-04-02T05:01:19.429 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:01:19.429 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:01:19.429 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:01:19.429 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:01:19.429 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:01:19.429 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:01:19.429 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:01:19.430 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:01:19.430 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:01:19.430 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Traceback (most recent call last): 2024-04-02T05:01:19.430 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T05:01:19.430 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T05:01:19.430 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T05:01:19.430 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T05:01:19.431 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T05:01:19.431 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: return self.event_loop.get_result(coro, timeout) 2024-04-02T05:01:19.431 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T05:01:19.431 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: return future.result(timeout) 2024-04-02T05:01:19.431 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T05:01:19.431 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: return self.__get_result() 2024-04-02T05:01:19.431 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T05:01:19.431 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: raise self._exception 2024-04-02T05:01:19.431 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T05:01:19.431 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: config_blobs=daemon_spec.final_config, 2024-04-02T05:01:19.431 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T05:01:19.431 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T05:01:19.431 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:01:19.431 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Creating ceph-iscsi config... 2024-04-02T05:01:19.432 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:01:19.432 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:01:19.432 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:01:19.432 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:01:19.432 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:01:19.432 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: Traceback (most recent call last): 2024-04-02T05:01:19.432 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:01:19.432 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: return _run_code(code, main_globals, None, 2024-04-02T05:01:19.432 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:01:19.432 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: exec(code, run_globals) 2024-04-02T05:01:19.432 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:01:19.432 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:01:19.432 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:01:19.432 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:01:19.432 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:01:19.433 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:01:19.433 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:01:19.433 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:01:19.433 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:01:19.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:19 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:01:19.473+0000 7f4ad47f0200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-02T05:01:19.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:19 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:01:19.617+0000 7f4ad47f0200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-02T05:01:20.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:20 smithi192 ceph-mon[25697]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:01:20.178 INFO:journalctl@ceph.mon.b.smithi192.stdout: Creating ceph-iscsi config... 2024-04-02T05:01:20.178 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:01:20.178 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:01:20.179 INFO:journalctl@ceph.mon.b.smithi192.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:01:20.179 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:01:20.179 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:01:20.179 INFO:journalctl@ceph.mon.b.smithi192.stdout: Traceback (most recent call last): 2024-04-02T05:01:20.179 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:01:20.179 INFO:journalctl@ceph.mon.b.smithi192.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:01:20.179 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:01:20.179 INFO:journalctl@ceph.mon.b.smithi192.stdout: exec(code, run_globals) 2024-04-02T05:01:20.179 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:01:20.179 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:01:20.179 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:01:20.180 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:01:20.180 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:01:20.180 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:01:20.180 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:01:20.180 INFO:journalctl@ceph.mon.b.smithi192.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:01:20.180 INFO:journalctl@ceph.mon.b.smithi192.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:01:20.180 INFO:journalctl@ceph.mon.b.smithi192.stdout: Traceback (most recent call last): 2024-04-02T05:01:20.180 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T05:01:20.180 INFO:journalctl@ceph.mon.b.smithi192.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T05:01:20.181 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T05:01:20.181 INFO:journalctl@ceph.mon.b.smithi192.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T05:01:20.181 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T05:01:20.181 INFO:journalctl@ceph.mon.b.smithi192.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-02T05:01:20.181 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T05:01:20.181 INFO:journalctl@ceph.mon.b.smithi192.stdout: return future.result(timeout) 2024-04-02T05:01:20.181 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T05:01:20.181 INFO:journalctl@ceph.mon.b.smithi192.stdout: return self.__get_result() 2024-04-02T05:01:20.181 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T05:01:20.181 INFO:journalctl@ceph.mon.b.smithi192.stdout: raise self._exception 2024-04-02T05:01:20.181 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T05:01:20.181 INFO:journalctl@ceph.mon.b.smithi192.stdout: config_blobs=daemon_spec.final_config, 2024-04-02T05:01:20.181 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T05:01:20.182 INFO:journalctl@ceph.mon.b.smithi192.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T05:01:20.182 INFO:journalctl@ceph.mon.b.smithi192.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:01:20.182 INFO:journalctl@ceph.mon.b.smithi192.stdout: Creating ceph-iscsi config... 2024-04-02T05:01:20.182 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:01:20.182 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:01:20.182 INFO:journalctl@ceph.mon.b.smithi192.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:01:20.182 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:01:20.182 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:01:20.182 INFO:journalctl@ceph.mon.b.smithi192.stdout: Traceback (most recent call last): 2024-04-02T05:01:20.182 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:01:20.183 INFO:journalctl@ceph.mon.b.smithi192.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:01:20.183 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:01:20.183 INFO:journalctl@ceph.mon.b.smithi192.stdout: exec(code, run_globals) 2024-04-02T05:01:20.183 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:01:20.183 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:01:20.183 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:01:20.183 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:01:20.183 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:01:20.183 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:01:20.183 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:01:20.183 INFO:journalctl@ceph.mon.b.smithi192.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:01:20.184 INFO:journalctl@ceph.mon.b.smithi192.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:01:20.292 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:20 smithi086 ceph-mon[29505]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:01:20.292 INFO:journalctl@ceph.mon.c.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T05:01:20.292 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:01:20.292 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:01:20.293 INFO:journalctl@ceph.mon.c.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:01:20.293 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:01:20.293 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:01:20.293 INFO:journalctl@ceph.mon.c.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:01:20.293 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:01:20.293 INFO:journalctl@ceph.mon.c.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:01:20.293 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:01:20.293 INFO:journalctl@ceph.mon.c.smithi086.stdout: exec(code, run_globals) 2024-04-02T05:01:20.293 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:01:20.293 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:01:20.293 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:01:20.293 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:01:20.294 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:01:20.294 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:01:20.294 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:01:20.294 INFO:journalctl@ceph.mon.c.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:01:20.294 INFO:journalctl@ceph.mon.c.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:01:20.294 INFO:journalctl@ceph.mon.c.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:01:20.294 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T05:01:20.294 INFO:journalctl@ceph.mon.c.smithi086.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T05:01:20.294 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T05:01:20.294 INFO:journalctl@ceph.mon.c.smithi086.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T05:01:20.294 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T05:01:20.295 INFO:journalctl@ceph.mon.c.smithi086.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-02T05:01:20.295 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T05:01:20.295 INFO:journalctl@ceph.mon.c.smithi086.stdout: return future.result(timeout) 2024-04-02T05:01:20.295 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T05:01:20.295 INFO:journalctl@ceph.mon.c.smithi086.stdout: return self.__get_result() 2024-04-02T05:01:20.295 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T05:01:20.295 INFO:journalctl@ceph.mon.c.smithi086.stdout: raise self._exception 2024-04-02T05:01:20.295 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T05:01:20.295 INFO:journalctl@ceph.mon.c.smithi086.stdout: config_blobs=daemon_spec.final_config, 2024-04-02T05:01:20.295 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T05:01:20.295 INFO:journalctl@ceph.mon.c.smithi086.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T05:01:20.295 INFO:journalctl@ceph.mon.c.smithi086.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:01:20.295 INFO:journalctl@ceph.mon.c.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T05:01:20.296 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:01:20.296 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:01:20.296 INFO:journalctl@ceph.mon.c.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:01:20.296 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:01:20.296 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:01:20.296 INFO:journalctl@ceph.mon.c.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:01:20.296 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:01:20.296 INFO:journalctl@ceph.mon.c.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:01:20.296 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:01:20.296 INFO:journalctl@ceph.mon.c.smithi086.stdout: exec(code, run_globals) 2024-04-02T05:01:20.296 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:01:20.296 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:01:20.296 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:01:20.297 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:01:20.297 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:01:20.297 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:01:20.297 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:01:20.297 INFO:journalctl@ceph.mon.c.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:01:20.297 INFO:journalctl@ceph.mon.c.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:01:20.297 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:20 smithi086 ceph-mon[24811]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:01:20.297 INFO:journalctl@ceph.mon.a.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T05:01:20.297 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:01:20.298 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:01:20.298 INFO:journalctl@ceph.mon.a.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:01:20.298 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:01:20.298 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:01:20.298 INFO:journalctl@ceph.mon.a.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:01:20.298 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:01:20.298 INFO:journalctl@ceph.mon.a.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:01:20.298 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:01:20.298 INFO:journalctl@ceph.mon.a.smithi086.stdout: exec(code, run_globals) 2024-04-02T05:01:20.298 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:01:20.298 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:01:20.299 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:01:20.299 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:01:20.299 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:01:20.299 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:01:20.299 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:01:20.299 INFO:journalctl@ceph.mon.a.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:01:20.299 INFO:journalctl@ceph.mon.a.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:01:20.299 INFO:journalctl@ceph.mon.a.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:01:20.299 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T05:01:20.299 INFO:journalctl@ceph.mon.a.smithi086.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T05:01:20.299 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T05:01:20.299 INFO:journalctl@ceph.mon.a.smithi086.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T05:01:20.299 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T05:01:20.300 INFO:journalctl@ceph.mon.a.smithi086.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-02T05:01:20.300 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T05:01:20.300 INFO:journalctl@ceph.mon.a.smithi086.stdout: return future.result(timeout) 2024-04-02T05:01:20.300 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T05:01:20.300 INFO:journalctl@ceph.mon.a.smithi086.stdout: return self.__get_result() 2024-04-02T05:01:20.300 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T05:01:20.300 INFO:journalctl@ceph.mon.a.smithi086.stdout: raise self._exception 2024-04-02T05:01:20.300 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T05:01:20.300 INFO:journalctl@ceph.mon.a.smithi086.stdout: config_blobs=daemon_spec.final_config, 2024-04-02T05:01:20.300 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T05:01:20.300 INFO:journalctl@ceph.mon.a.smithi086.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T05:01:20.300 INFO:journalctl@ceph.mon.a.smithi086.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:01:20.301 INFO:journalctl@ceph.mon.a.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T05:01:20.301 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:01:20.301 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:01:20.301 INFO:journalctl@ceph.mon.a.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:01:20.301 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:01:20.301 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:01:20.301 INFO:journalctl@ceph.mon.a.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:01:20.301 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:01:20.301 INFO:journalctl@ceph.mon.a.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:01:20.301 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:01:20.301 INFO:journalctl@ceph.mon.a.smithi086.stdout: exec(code, run_globals) 2024-04-02T05:01:20.301 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:01:20.302 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:01:20.302 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:01:20.302 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:01:20.302 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:01:20.302 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:01:20.302 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:01:20.302 INFO:journalctl@ceph.mon.a.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:01:20.302 INFO:journalctl@ceph.mon.a.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:01:20.302 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:19 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:01:19.980+0000 7f4ad47f0200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-02T05:01:20.303 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:01:20.080+0000 7f4ad47f0200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-02T05:01:20.303 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:01:20.287+0000 7f4ad47f0200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-02T05:01:20.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:01:20.485+0000 7f4ad47f0200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-02T05:01:21.032 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:01:20.789+0000 7f4ad47f0200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-02T05:01:21.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:21 smithi086 ceph-mon[29505]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:01:21.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:21 smithi086 ceph-mon[24811]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:01:21.334 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:21 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:01:21.075+0000 7f4ad47f0200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-02T05:01:21.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:21 smithi192 ceph-mon[25697]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:01:21.655 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:21 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:01:21.335+0000 7f4ad47f0200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-02T05:01:22.318 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:22 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:01:22.195+0000 7f4ad47f0200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-02T05:01:23.083 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:22 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:01:22.808+0000 7f4ad47f0200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-02T05:01:23.084 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:22 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:01:22.918+0000 7f4ad47f0200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-02T05:01:23.582 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:01:23.151+0000 7f4ad47f0200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-02T05:01:23.582 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: [02/Apr/2024:05:01:23] ENGINE Bus STARTING 2024-04-02T05:01:23.582 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: CherryPy Checker: 2024-04-02T05:01:23.582 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: The Application mounted at '' has an empty config. 2024-04-02T05:01:23.582 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:01:23.582 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: [02/Apr/2024:05:01:23] ENGINE Serving on http://:::9283 2024-04-02T05:01:23.582 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: [02/Apr/2024:05:01:23] ENGINE Bus STARTED 2024-04-02T05:01:23.582 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-mon[24811]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:01:23.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:23.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:23.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:23.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:23.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:01:23.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:01:23.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:23.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:01:23.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:01:23.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:01:23.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-mon[24811]: Standby manager daemon y restarted 2024-04-02T05:01:23.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-mon[24811]: Standby manager daemon y started 2024-04-02T05:01:23.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-mon[24811]: from='mgr.? 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-02T05:01:23.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-mon[24811]: from='mgr.? 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T05:01:23.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-mon[24811]: from='mgr.? 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-02T05:01:23.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-mon[24811]: from='mgr.? 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T05:01:23.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-mon[29505]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:01:23.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:23.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:23.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:23.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:23.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:01:23.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:01:23.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:23.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:01:23.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:01:23.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:01:23.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-mon[29505]: Standby manager daemon y restarted 2024-04-02T05:01:23.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-mon[29505]: Standby manager daemon y started 2024-04-02T05:01:23.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-02T05:01:23.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T05:01:23.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-02T05:01:23.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:23 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T05:01:23.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:23 smithi192 ceph-mon[25697]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:01:23.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:23 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:23.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:23 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:23.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:23 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:23.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:23 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:23.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:23 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:01:23.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:23 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:01:23.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:23 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:23.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:23 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:01:23.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:23 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:01:23.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:23 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:01:23.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:23 smithi192 ceph-mon[25697]: Standby manager daemon y restarted 2024-04-02T05:01:23.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:23 smithi192 ceph-mon[25697]: Standby manager daemon y started 2024-04-02T05:01:23.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:23 smithi192 ceph-mon[25697]: from='mgr.? 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-02T05:01:23.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:23 smithi192 ceph-mon[25697]: from='mgr.? 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T05:01:23.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:23 smithi192 ceph-mon[25697]: from='mgr.? 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-02T05:01:23.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:23 smithi192 ceph-mon[25697]: from='mgr.? 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T05:01:24.508 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:24 smithi086 ceph-mon[29505]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 953 B/s rd, 1 op/s 2024-04-02T05:01:24.508 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:24 smithi086 ceph-mon[29505]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 586 B/s rd, 0 op/s 2024-04-02T05:01:24.509 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:24 smithi086 ceph-mon[29505]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T05:01:24.509 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:24 smithi086 ceph-mon[29505]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T05:01:24.509 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:24 smithi086 ceph-mon[29505]: Health check update: 2 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-02T05:01:24.509 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:24 smithi086 ceph-mon[29505]: mgrmap e26: x(active, since 3m), standbys: y 2024-04-02T05:01:24.509 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:24 smithi086 ceph-mon[24811]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 953 B/s rd, 1 op/s 2024-04-02T05:01:24.509 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:24 smithi086 ceph-mon[24811]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 586 B/s rd, 0 op/s 2024-04-02T05:01:24.509 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:24 smithi086 ceph-mon[24811]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T05:01:24.509 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:24 smithi086 ceph-mon[24811]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T05:01:24.509 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:24 smithi086 ceph-mon[24811]: Health check update: 2 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-02T05:01:24.509 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:24 smithi086 ceph-mon[24811]: mgrmap e26: x(active, since 3m), standbys: y 2024-04-02T05:01:24.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:24 smithi192 ceph-mon[25697]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 953 B/s rd, 1 op/s 2024-04-02T05:01:24.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:24 smithi192 ceph-mon[25697]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 586 B/s rd, 0 op/s 2024-04-02T05:01:24.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:24 smithi192 ceph-mon[25697]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T05:01:24.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:24 smithi192 ceph-mon[25697]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T05:01:24.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:24 smithi192 ceph-mon[25697]: Health check update: 2 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-02T05:01:24.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:24 smithi192 ceph-mon[25697]: mgrmap e26: x(active, since 3m), standbys: y 2024-04-02T05:01:26.463 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:26 smithi086 ceph-mon[24811]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 586 B/s rd, 0 op/s 2024-04-02T05:01:26.463 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:26 smithi086 ceph-mon[29505]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 586 B/s rd, 0 op/s 2024-04-02T05:01:26.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:26 smithi192 ceph-mon[25697]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 586 B/s rd, 0 op/s 2024-04-02T05:01:27.579 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:27 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:01:27] "GET /metrics HTTP/1.1" 200 37977 "" "Prometheus/2.33.4" 2024-04-02T05:01:28.460 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:28 smithi086 ceph-mon[24811]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 586 B/s rd, 0 op/s 2024-04-02T05:01:28.460 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:28 smithi086 ceph-mon[29505]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 586 B/s rd, 0 op/s 2024-04-02T05:01:28.590 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:28 smithi192 ceph-mon[25697]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 586 B/s rd, 0 op/s 2024-04-02T05:01:28.928 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:01:28 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T05:01:28.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"smithi192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"172.21.15.192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-02T05:01:30.329 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:30 smithi086 ceph-mon[29505]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:01:30.329 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:30 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/2332625468' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T05:01:30.329 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:30 smithi086 ceph-mon[24811]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:01:30.330 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:30 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/2332625468' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T05:01:30.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:30 smithi192 ceph-mon[25697]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:01:30.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:30 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/2332625468' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T05:01:31.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:31 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:31.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:31 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:31.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:31 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T05:01:31.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:31 smithi192 ceph-mon[25697]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T05:01:31.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:31 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:31.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:31 smithi192 ceph-mon[25697]: Adding iSCSI gateway http://:@172.21.15.86:5000 to Dashboard 2024-04-02T05:01:31.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:31 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T05:01:31.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:31 smithi192 ceph-mon[25697]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T05:01:31.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:31 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi086"}]: dispatch 2024-04-02T05:01:31.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:31 smithi192 ceph-mon[25697]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi086"}]: dispatch 2024-04-02T05:01:31.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:31 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:31.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:31 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:31.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:31 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:31.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:31 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T05:01:31.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:31 smithi086 ceph-mon[24811]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T05:01:31.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:31 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:31.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:31 smithi086 ceph-mon[24811]: Adding iSCSI gateway http://:@172.21.15.86:5000 to Dashboard 2024-04-02T05:01:31.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:31 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T05:01:31.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:31 smithi086 ceph-mon[24811]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T05:01:31.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:31 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi086"}]: dispatch 2024-04-02T05:01:31.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:31 smithi086 ceph-mon[24811]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi086"}]: dispatch 2024-04-02T05:01:31.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:31 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:31.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:31 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:31.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:31 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:31.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:31 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T05:01:31.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:31 smithi086 ceph-mon[29505]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T05:01:31.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:31 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:31.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:31 smithi086 ceph-mon[29505]: Adding iSCSI gateway http://:@172.21.15.86:5000 to Dashboard 2024-04-02T05:01:31.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:31 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T05:01:31.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:31 smithi086 ceph-mon[29505]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T05:01:31.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:31 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi086"}]: dispatch 2024-04-02T05:01:31.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:31 smithi086 ceph-mon[29505]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi086"}]: dispatch 2024-04-02T05:01:31.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:31 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:32.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:32 smithi192 ceph-mon[25697]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:01:32.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:32 smithi086 ceph-mon[24811]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:01:32.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:32 smithi086 ceph-mon[29505]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:01:33.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:33 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:01:33.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:33 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:01:33.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:33 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:01:34.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:34 smithi192 ceph-mon[25697]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:01:34.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:34 smithi086 ceph-mon[24811]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:01:34.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:34 smithi086 ceph-mon[29505]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:01:36.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:36 smithi192 ceph-mon[25697]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-02T05:01:36.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:36 smithi086 ceph-mon[24811]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-02T05:01:36.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:36 smithi086 ceph-mon[29505]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-02T05:01:37.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:01:37] "GET /metrics HTTP/1.1" 200 37975 "" "Prometheus/2.33.4" 2024-04-02T05:01:38.663 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:38 smithi192 ceph-mon[25697]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-02T05:01:38.717 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:38 smithi086 ceph-mon[29505]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-02T05:01:38.717 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:38 smithi086 ceph-mon[24811]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-02T05:01:38.928 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:01:38 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T05:01:38.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"smithi192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"172.21.15.192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-02T05:01:40.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:40 smithi086 ceph-mon[24811]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-02T05:01:40.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:40 smithi086 ceph-mon[29505]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-02T05:01:40.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:40 smithi192 ceph-mon[25697]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-02T05:01:41.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:41 smithi192 ceph-mon[25697]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:01:41.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:41 smithi086 ceph-mon[29505]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:01:41.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:41 smithi086 ceph-mon[24811]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:01:42.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:42 smithi192 ceph-mon[25697]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:01:42.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:42 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:42.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:42 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:42.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:42 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:01:42.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:42 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:01:42.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:42 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:42.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:42 smithi086 ceph-mon[29505]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:01:42.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:42 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:42.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:42 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:42.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:42 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:01:42.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:42 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:01:42.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:42 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:42.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:42 smithi086 ceph-mon[24811]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:01:42.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:42 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:42.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:42 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:42.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:42 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:01:42.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:42 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:01:42.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:42 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:01:43.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:43 smithi192 ceph-mon[25697]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 919 B/s rd, 0 op/s 2024-04-02T05:01:43.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:43 smithi192 ceph-mon[25697]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:01:43.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:43 smithi192 ceph-mon[25697]: Health check update: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-02T05:01:43.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:43 smithi086 ceph-mon[29505]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 919 B/s rd, 0 op/s 2024-04-02T05:01:43.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:43 smithi086 ceph-mon[29505]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:01:43.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:43 smithi086 ceph-mon[29505]: Health check update: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-02T05:01:43.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:43 smithi086 ceph-mon[24811]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 919 B/s rd, 0 op/s 2024-04-02T05:01:43.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:43 smithi086 ceph-mon[24811]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:01:43.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:43 smithi086 ceph-mon[24811]: Health check update: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-02T05:01:45.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:45 smithi192 ceph-mon[25697]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:01:45.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:45 smithi086 ceph-mon[24811]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:01:45.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:45 smithi086 ceph-mon[29505]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:01:47.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:46 smithi086 ceph-mon[24811]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:01:47.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:46 smithi086 ceph-mon[29505]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:01:47.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:46 smithi192 ceph-mon[25697]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:01:47.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:01:47] "GET /metrics HTTP/1.1" 200 37975 "" "Prometheus/2.33.4" 2024-04-02T05:01:48.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:48 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:01:48.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:48 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:01:48.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:48 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:01:48.928 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:01:48 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T05:01:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"smithi192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"172.21.15.192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-02T05:01:49.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:49 smithi086 ceph-mon[24811]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:01:49.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:49 smithi086 ceph-mon[29505]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:01:49.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:49 smithi192 ceph-mon[25697]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:01:51.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:51 smithi086 ceph-mon[24811]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:01:51.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:51 smithi086 ceph-mon[24811]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:01:51.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:51 smithi086 ceph-mon[29505]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:01:51.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:51 smithi086 ceph-mon[29505]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:01:51.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:51 smithi192 ceph-mon[25697]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:01:51.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:51 smithi192 ceph-mon[25697]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:01:53.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:53 smithi086 ceph-mon[24811]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:01:53.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:53 smithi086 ceph-mon[29505]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:01:53.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:53 smithi192 ceph-mon[25697]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:01:55.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:55 smithi086 ceph-mon[29505]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:01:55.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:55 smithi086 ceph-mon[24811]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:01:55.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:55 smithi192 ceph-mon[25697]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:01:57.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:56 smithi086 ceph-mon[29505]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:01:57.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:56 smithi086 ceph-mon[24811]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:01:57.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:56 smithi192 ceph-mon[25697]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:01:57.584 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:01:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:01:57] "GET /metrics HTTP/1.1" 200 37977 "" "Prometheus/2.33.4" 2024-04-02T05:01:58.928 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:01:58 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T05:01:58.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"smithi192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"172.21.15.192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-02T05:01:59.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:01:59 smithi086 ceph-mon[24811]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:01:59.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:01:59 smithi086 ceph-mon[29505]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:01:59.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:01:59 smithi192 ceph-mon[25697]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:01.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:01 smithi086 ceph-mon[24811]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:02:01.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:01 smithi086 ceph-mon[24811]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:02:01.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:01 smithi086 ceph-mon[29505]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:02:01.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:01 smithi086 ceph-mon[29505]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:02:01.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:01 smithi192 ceph-mon[25697]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:02:01.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:01 smithi192 ceph-mon[25697]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:02:03.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:03 smithi086 ceph-mon[24811]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:03.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:03 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:02:03.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:03 smithi086 ceph-mon[29505]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:03.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:03 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:02:03.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:03 smithi192 ceph-mon[25697]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:03.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:03 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:02:05.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:05 smithi086 ceph-mon[24811]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:02:05.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:05 smithi086 ceph-mon[29505]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:02:05.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:05 smithi192 ceph-mon[25697]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:02:07.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:06 smithi086 ceph-mon[24811]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:07.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:06 smithi086 ceph-mon[29505]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:07.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:06 smithi192 ceph-mon[25697]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:07.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:02:07 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:02:07] "GET /metrics HTTP/1.1" 200 37974 "" "Prometheus/2.33.4" 2024-04-02T05:02:08.928 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:02:08 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T05:02:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"smithi192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"172.21.15.192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-02T05:02:09.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:09 smithi086 ceph-mon[24811]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:09.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:09 smithi086 ceph-mon[29505]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:09.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:09 smithi192 ceph-mon[25697]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:11.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:11 smithi086 ceph-mon[24811]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:02:11.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:11 smithi086 ceph-mon[24811]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:02:11.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:11 smithi086 ceph-mon[29505]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:02:11.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:11 smithi086 ceph-mon[29505]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:02:11.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:11 smithi192 ceph-mon[25697]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:02:11.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:11 smithi192 ceph-mon[25697]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:02:13.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:13 smithi086 ceph-mon[29505]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:13.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:13 smithi086 ceph-mon[24811]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:13.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:13 smithi192 ceph-mon[25697]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:15.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:15 smithi086 ceph-mon[29505]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:02:15.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:15 smithi086 ceph-mon[24811]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:02:15.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:15 smithi192 ceph-mon[25697]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:02:17.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:16 smithi086 ceph-mon[24811]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:17.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:16 smithi086 ceph-mon[29505]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:17.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:16 smithi192 ceph-mon[25697]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:17.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:02:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:02:17] "GET /metrics HTTP/1.1" 200 37974 "" "Prometheus/2.33.4" 2024-04-02T05:02:18.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:18 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:02:18.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:18 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:02:18.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:18 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:02:18.928 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:02:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T05:02:18.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"smithi192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"172.21.15.192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-02T05:02:19.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:19 smithi086 ceph-mon[24811]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:19.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:19 smithi086 ceph-mon[29505]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:19.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:19 smithi192 ceph-mon[25697]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:21.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:21 smithi086 ceph-mon[24811]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:02:21.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:21 smithi086 ceph-mon[24811]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:02:21.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:21 smithi086 ceph-mon[29505]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:02:21.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:21 smithi086 ceph-mon[29505]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:02:21.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:21 smithi192 ceph-mon[25697]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:02:21.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:21 smithi192 ceph-mon[25697]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:02:23.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:23 smithi086 ceph-mon[24811]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:23.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:23 smithi086 ceph-mon[29505]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:23.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:23 smithi192 ceph-mon[25697]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:25.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:25 smithi086 ceph-mon[24811]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:02:25.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:25 smithi086 ceph-mon[29505]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:02:25.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:25 smithi192 ceph-mon[25697]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:02:27.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:26 smithi086 ceph-mon[24811]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:27.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:26 smithi086 ceph-mon[29505]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:27.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:26 smithi192 ceph-mon[25697]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:27.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:02:27 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:02:27] "GET /metrics HTTP/1.1" 200 37975 "" "Prometheus/2.33.4" 2024-04-02T05:02:28.928 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:02:28 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T05:02:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"smithi192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"172.21.15.192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-02T05:02:29.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:29 smithi086 ceph-mon[24811]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:29.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:29 smithi086 ceph-mon[29505]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:29.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:29 smithi192 ceph-mon[25697]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:31.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:31 smithi086 ceph-mon[29505]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:02:31.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:31 smithi086 ceph-mon[29505]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:02:31.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:31 smithi086 ceph-mon[24811]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:02:31.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:31 smithi086 ceph-mon[24811]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:02:31.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:31 smithi192 ceph-mon[25697]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:02:31.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:31 smithi192 ceph-mon[25697]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:02:33.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:33 smithi086 ceph-mon[24811]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:33.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:33 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:02:33.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:33 smithi086 ceph-mon[29505]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:33.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:33 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:02:33.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:33 smithi192 ceph-mon[25697]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:33.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:33 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:02:35.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:35 smithi086 ceph-mon[29505]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:02:35.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:35 smithi086 ceph-mon[24811]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:02:35.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:35 smithi192 ceph-mon[25697]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:02:37.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:36 smithi086 ceph-mon[29505]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:37.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:36 smithi086 ceph-mon[24811]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:37.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:36 smithi192 ceph-mon[25697]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:37.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:02:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:02:37] "GET /metrics HTTP/1.1" 200 37969 "" "Prometheus/2.33.4" 2024-04-02T05:02:38.928 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:02:38 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T05:02:38.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"smithi192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"172.21.15.192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-02T05:02:39.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:39 smithi086 ceph-mon[24811]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:39.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:39 smithi086 ceph-mon[29505]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:39.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:39 smithi192 ceph-mon[25697]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:41.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:41 smithi086 ceph-mon[24811]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:02:41.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:41 smithi086 ceph-mon[24811]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:02:41.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:41 smithi086 ceph-mon[29505]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:02:41.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:41 smithi086 ceph-mon[29505]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:02:41.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:41 smithi192 ceph-mon[25697]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:02:41.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:41 smithi192 ceph-mon[25697]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:02:43.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:43 smithi086 ceph-mon[24811]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:43.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:43 smithi086 ceph-mon[29505]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:43.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:43 smithi192 ceph-mon[25697]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:45.325 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:45 smithi086 ceph-mon[29505]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:02:45.325 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:45 smithi086 ceph-mon[24811]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:02:45.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:45 smithi192 ceph-mon[25697]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:02:47.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:47 smithi086 ceph-mon[24811]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:47.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:47 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:02:47.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:47 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:02:47.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:47 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:02:47.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:47 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:02:47.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:47 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:02:47.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:47 smithi086 ceph-mon[29505]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:47.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:47 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:02:47.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:47 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:02:47.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:47 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:02:47.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:47 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:02:47.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:47 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:02:47.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:02:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:02:47] "GET /metrics HTTP/1.1" 200 37969 "" "Prometheus/2.33.4" 2024-04-02T05:02:47.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:47 smithi192 ceph-mon[25697]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:02:47.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:47 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:02:47.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:47 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:02:47.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:47 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:02:47.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:47 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:02:47.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:47 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:02:48.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:48 smithi086 ceph-mon[24811]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 940 B/s rd, 0 op/s 2024-04-02T05:02:48.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:48 smithi086 ceph-mon[24811]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:02:48.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:48 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:02:48.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:48 smithi086 ceph-mon[29505]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 940 B/s rd, 0 op/s 2024-04-02T05:02:48.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:48 smithi086 ceph-mon[29505]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:02:48.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:48 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:02:48.662 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:48 smithi192 ceph-mon[25697]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 940 B/s rd, 0 op/s 2024-04-02T05:02:48.662 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:48 smithi192 ceph-mon[25697]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:02:48.662 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:48 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:02:48.928 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:02:48 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T05:02:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"smithi192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"172.21.15.192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-02T05:02:50.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:50 smithi086 ceph-mon[24811]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 576 B/s rd, 0 op/s 2024-04-02T05:02:50.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:50 smithi086 ceph-mon[29505]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 576 B/s rd, 0 op/s 2024-04-02T05:02:50.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:50 smithi192 ceph-mon[25697]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 576 B/s rd, 0 op/s 2024-04-02T05:02:51.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:51 smithi086 ceph-mon[24811]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:02:51.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:51 smithi086 ceph-mon[29505]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:02:51.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:51 smithi192 ceph-mon[25697]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:02:52.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:52 smithi086 ceph-mon[24811]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:02:52.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:52 smithi086 ceph-mon[29505]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:02:52.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:52 smithi192 ceph-mon[25697]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:02:54.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:54 smithi086 ceph-mon[29505]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 576 B/s rd, 0 op/s 2024-04-02T05:02:54.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:54 smithi086 ceph-mon[24811]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 576 B/s rd, 0 op/s 2024-04-02T05:02:54.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:54 smithi192 ceph-mon[25697]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 576 B/s rd, 0 op/s 2024-04-02T05:02:56.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:56 smithi086 ceph-mon[24811]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:02:56.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:56 smithi086 ceph-mon[29505]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:02:56.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:56 smithi192 ceph-mon[25697]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:02:57.579 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:02:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:02:57] "GET /metrics HTTP/1.1" 200 37975 "" "Prometheus/2.33.4" 2024-04-02T05:02:58.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:02:58 smithi086 ceph-mon[24811]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:02:58.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:02:58 smithi086 ceph-mon[29505]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:02:58.662 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:02:58 smithi192 ceph-mon[25697]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:02:58.927 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:02:58 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T05:02:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"smithi192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.86\", device_class=\"ssd\", hostname=\"smithi086\", instance=\"172.21.15.192:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.86\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-02T05:03:00.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:00 smithi086 ceph-mon[29505]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:00.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:00 smithi086 ceph-mon[24811]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:00.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:00 smithi192 ceph-mon[25697]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:01.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:01 smithi086 ceph-mon[29505]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:03:01.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:01 smithi086 ceph-mon[24811]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:03:01.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:01 smithi192 ceph-mon[25697]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:03:02.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:02 smithi192 ceph-mon[25697]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:03:02.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:02 smithi086 ceph-mon[24811]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:03:02.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:02 smithi086 ceph-mon[29505]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:03:03.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:03 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:03:03.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:03 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:03:03.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:03 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:03:04.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:04 smithi192 ceph-mon[25697]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:04.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:04 smithi086 ceph-mon[24811]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:04.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:04 smithi086 ceph-mon[29505]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:06.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:06 smithi192 ceph-mon[25697]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:03:06.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:06 smithi086 ceph-mon[24811]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:03:06.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:06 smithi086 ceph-mon[29505]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:03:07.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:03:07 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:03:07] "GET /metrics HTTP/1.1" 200 37972 "" "Prometheus/2.33.4" 2024-04-02T05:03:08.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:08 smithi192 ceph-mon[25697]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:08.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:08 smithi086 ceph-mon[29505]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:08.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:08 smithi086 ceph-mon[24811]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:10.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:10 smithi192 ceph-mon[25697]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:10.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:10 smithi086 ceph-mon[29505]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:10.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:10 smithi086 ceph-mon[24811]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:11.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:11 smithi192 ceph-mon[25697]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:03:11.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:11 smithi086 ceph-mon[24811]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:03:11.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:11 smithi086 ceph-mon[29505]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:03:12.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:12 smithi192 ceph-mon[25697]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:03:12.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:12 smithi086 ceph-mon[24811]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:03:12.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:12 smithi086 ceph-mon[29505]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:03:14.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:14 smithi192 ceph-mon[25697]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:14.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:14 smithi086 ceph-mon[24811]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:14.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:14 smithi086 ceph-mon[29505]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:16.635 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:16 smithi086 ceph-mon[24811]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:03:16.636 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:16 smithi086 ceph-mon[29505]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:03:16.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:16 smithi192 ceph-mon[25697]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:03:17.678 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:03:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:03:17] "GET /metrics HTTP/1.1" 200 37972 "" "Prometheus/2.33.4" 2024-04-02T05:03:18.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:18 smithi192 ceph-mon[25697]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:18.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:18 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:03:18.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:18 smithi086 ceph-mon[24811]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:18.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:18 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:03:18.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:18 smithi086 ceph-mon[29505]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:18.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:18 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:03:20.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:20 smithi192 ceph-mon[25697]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:20.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:20 smithi086 ceph-mon[29505]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:20.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:20 smithi086 ceph-mon[24811]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:21.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:21 smithi192 ceph-mon[25697]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:03:21.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:21 smithi086 ceph-mon[24811]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:03:21.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:21 smithi086 ceph-mon[29505]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:03:22.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:22 smithi192 ceph-mon[25697]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:03:22.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:22 smithi086 ceph-mon[24811]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:03:22.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:22 smithi086 ceph-mon[29505]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:03:24.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:24 smithi192 ceph-mon[25697]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:24.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:24 smithi086 ceph-mon[24811]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:24.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:24 smithi086 ceph-mon[29505]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:26.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:26 smithi192 ceph-mon[25697]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:03:26.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:26 smithi086 ceph-mon[24811]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:03:26.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:26 smithi086 ceph-mon[29505]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:03:27.579 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:03:27 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:03:27] "GET /metrics HTTP/1.1" 200 37970 "" "Prometheus/2.33.4" 2024-04-02T05:03:28.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:28 smithi192 ceph-mon[25697]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:28.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:28 smithi086 ceph-mon[24811]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:28.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:28 smithi086 ceph-mon[29505]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:30.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:30 smithi192 ceph-mon[25697]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:30.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:30 smithi086 ceph-mon[29505]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:30.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:30 smithi086 ceph-mon[24811]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:31.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:31 smithi192 ceph-mon[25697]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:03:31.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:31 smithi086 ceph-mon[24811]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:03:31.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:31 smithi086 ceph-mon[29505]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:03:32.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:32 smithi192 ceph-mon[25697]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:03:32.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:32 smithi086 ceph-mon[24811]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:03:32.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:32 smithi086 ceph-mon[29505]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:03:33.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:33 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:03:33.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:33 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:03:33.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:33 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:03:34.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:34 smithi192 ceph-mon[25697]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:34.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:34 smithi086 ceph-mon[24811]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:34.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:34 smithi086 ceph-mon[29505]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:36.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:36 smithi192 ceph-mon[25697]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:03:36.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:36 smithi086 ceph-mon[24811]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:03:36.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:36 smithi086 ceph-mon[29505]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:03:37.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:03:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:03:37] "GET /metrics HTTP/1.1" 200 37972 "" "Prometheus/2.33.4" 2024-04-02T05:03:38.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:38 smithi192 ceph-mon[25697]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:38.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:38 smithi086 ceph-mon[24811]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:38.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:38 smithi086 ceph-mon[29505]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:40.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:40 smithi192 ceph-mon[25697]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:40.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:40 smithi086 ceph-mon[24811]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:40.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:40 smithi086 ceph-mon[29505]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:41.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:41 smithi192 ceph-mon[25697]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:03:41.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:41 smithi086 ceph-mon[24811]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:03:41.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:41 smithi086 ceph-mon[29505]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:03:42.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:42 smithi192 ceph-mon[25697]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:03:42.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:42 smithi086 ceph-mon[24811]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:03:42.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:42 smithi086 ceph-mon[29505]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:03:43.638 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch ps' 2024-04-02T05:03:44.535 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:44 smithi086 ceph-mon[29505]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:44.535 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:44 smithi086 ceph-mon[24811]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:44.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:44 smithi192 ceph-mon[25697]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:03:45.542 INFO:teuthology.orchestra.run.smithi086.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T05:03:45.543 INFO:teuthology.orchestra.run.smithi086.stdout:alertmanager.a smithi086 *:9093,9094 running (5m) 59s ago 13m 23.6M - 0.23.0 ba2b418f427c b2769c8e287a 2024-04-02T05:03:45.543 INFO:teuthology.orchestra.run.smithi086.stdout:grafana.a smithi192 *:3000 error 2m ago 13m - - 2024-04-02T05:03:45.543 INFO:teuthology.orchestra.run.smithi086.stdout:iscsi.foo.smithi086.rsbqwx smithi086 running (2m) 59s ago 12m 48.6M - 3.5 e1d6a67b021e 9777e4d9dbb9 2024-04-02T05:03:45.543 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.x smithi192 *:8443,9283 running (8m) 2m ago 18m 511M - 19.0.0-1979-g6cec793d 78f830190839 fd618149d6b4 2024-04-02T05:03:45.543 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.y smithi086 *:8443,9283,8765 running (2m) 59s ago 19m 451M - 19.0.0-1979-g6cec793d 78f830190839 3dca54a018ab 2024-04-02T05:03:45.543 INFO:teuthology.orchestra.run.smithi086.stdout:mon.a smithi086 running (19m) 59s ago 19m 61.3M 2048M 17.2.0 e1d6a67b021e b6c507ab96f8 2024-04-02T05:03:45.543 INFO:teuthology.orchestra.run.smithi086.stdout:mon.b smithi192 running (18m) 2m ago 18m 51.3M 2048M 17.2.0 e1d6a67b021e 4dadd8d1c318 2024-04-02T05:03:45.543 INFO:teuthology.orchestra.run.smithi086.stdout:mon.c smithi086 running (18m) 59s ago 18m 40.7M 2048M 17.2.0 e1d6a67b021e 49275483a2d4 2024-04-02T05:03:45.543 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.a smithi086 *:9100 running (5m) 59s ago 14m 19.9M - 1.3.1 1dbe0e931976 843a5e9887fb 2024-04-02T05:03:45.544 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.b smithi192 *:9100 running (5m) 2m ago 14m 17.5M - 1.3.1 1dbe0e931976 bd849d6c4fb0 2024-04-02T05:03:45.544 INFO:teuthology.orchestra.run.smithi086.stdout:osd.0 smithi086 running (17m) 59s ago 17m 55.7M 2992M 17.2.0 e1d6a67b021e 6d90c60d1c71 2024-04-02T05:03:45.544 INFO:teuthology.orchestra.run.smithi086.stdout:osd.1 smithi086 running (17m) 59s ago 17m 54.1M 2992M 17.2.0 e1d6a67b021e 203689f71603 2024-04-02T05:03:45.544 INFO:teuthology.orchestra.run.smithi086.stdout:osd.2 smithi086 running (17m) 59s ago 17m 49.2M 2992M 17.2.0 e1d6a67b021e 472c860a9af8 2024-04-02T05:03:45.544 INFO:teuthology.orchestra.run.smithi086.stdout:osd.3 smithi086 running (16m) 59s ago 16m 51.6M 2992M 17.2.0 e1d6a67b021e 327538dac44e 2024-04-02T05:03:45.544 INFO:teuthology.orchestra.run.smithi086.stdout:osd.4 smithi192 running (16m) 2m ago 16m 52.8M 3248M 17.2.0 e1d6a67b021e 6535d7919a2d 2024-04-02T05:03:45.544 INFO:teuthology.orchestra.run.smithi086.stdout:osd.5 smithi192 running (15m) 2m ago 15m 53.8M 3248M 17.2.0 e1d6a67b021e 7259249605f2 2024-04-02T05:03:45.544 INFO:teuthology.orchestra.run.smithi086.stdout:osd.6 smithi192 running (15m) 2m ago 15m 49.8M 3248M 17.2.0 e1d6a67b021e bb7d171b02a1 2024-04-02T05:03:45.544 INFO:teuthology.orchestra.run.smithi086.stdout:osd.7 smithi192 running (15m) 2m ago 15m 52.9M 3248M 17.2.0 e1d6a67b021e 95f5a0353999 2024-04-02T05:03:45.544 INFO:teuthology.orchestra.run.smithi086.stdout:prometheus.a smithi192 *:9095 running (5m) 2m ago 14m 67.0M - 2.33.4 514e6a882f6e d3ec96f3bde3 2024-04-02T05:03:45.545 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi086.bxzxlp smithi086 *:8000 running (13m) 59s ago 13m 75.4M - 17.2.0 e1d6a67b021e 6725785fbc53 2024-04-02T05:03:45.545 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi192.pmlbot smithi192 *:8000 running (12m) 2m ago 12m 73.7M - 17.2.0 e1d6a67b021e 5ae3638f8b1c 2024-04-02T05:03:46.139 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph versions' 2024-04-02T05:03:46.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:46 smithi192 ceph-mon[25697]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:03:46.744 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:46 smithi086 ceph-mon[29505]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:03:46.745 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:46 smithi086 ceph-mon[24811]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:03:47.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:47 smithi192 ceph-mon[25697]: from='client.15219 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:03:47.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:03:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:03:47] "GET /metrics HTTP/1.1" 200 37972 "" "Prometheus/2.33.4" 2024-04-02T05:03:47.725 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:47 smithi086 ceph-mon[29505]: from='client.15219 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:03:47.725 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:47 smithi086 ceph-mon[24811]: from='client.15219 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:03:48.211 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:03:48.211 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": { 2024-04-02T05:03:48.212 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-02T05:03:48.212 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:03:48.212 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": { 2024-04-02T05:03:48.212 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 2 2024-04-02T05:03:48.212 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:03:48.212 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": { 2024-04-02T05:03:48.212 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-02T05:03:48.212 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:03:48.212 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": {}, 2024-04-02T05:03:48.212 INFO:teuthology.orchestra.run.smithi086.stdout: "rgw": { 2024-04-02T05:03:48.212 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-02T05:03:48.212 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:03:48.212 INFO:teuthology.orchestra.run.smithi086.stdout: "overall": { 2024-04-02T05:03:48.212 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-02T05:03:48.212 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 2 2024-04-02T05:03:48.213 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-04-02T05:03:48.213 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:03:48.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:48 smithi086 ceph-mon[24811]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-02T05:03:48.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:48 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:03:48.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:48 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:03:48.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:48 smithi086 ceph-mon[24811]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:03:48.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:48 smithi086 ceph-mon[24811]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:03:48.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:48 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1168177112' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:03:48.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:48 smithi086 ceph-mon[29505]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-02T05:03:48.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:48 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:03:48.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:48 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:03:48.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:48 smithi086 ceph-mon[29505]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:03:48.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:48 smithi086 ceph-mon[29505]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:03:48.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:48 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1168177112' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:03:48.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:48 smithi192 ceph-mon[25697]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-02T05:03:48.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:48 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:03:48.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:48 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:03:48.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:48 smithi192 ceph-mon[25697]: from='mgr.34691 ' entity='mgr.x' 2024-04-02T05:03:48.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:48 smithi192 ceph-mon[25697]: from='mgr.34691 172.21.15.192:0/1700387519' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:03:48.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:48 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1168177112' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:03:49.037 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph health detail' 2024-04-02T05:03:49.640 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:49 smithi086 ceph-mon[29505]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 857 B/s rd, 0 op/s 2024-04-02T05:03:49.640 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:49 smithi086 ceph-mon[29505]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T05:03:49.640 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:49 smithi086 ceph-mon[24811]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 857 B/s rd, 0 op/s 2024-04-02T05:03:49.640 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:49 smithi086 ceph-mon[24811]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T05:03:49.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:49 smithi192 ceph-mon[25697]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 857 B/s rd, 0 op/s 2024-04-02T05:03:49.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:49 smithi192 ceph-mon[25697]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T05:03:50.944 INFO:teuthology.orchestra.run.smithi086.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-02T05:03:50.944 INFO:teuthology.orchestra.run.smithi086.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-02T05:03:50.944 INFO:teuthology.orchestra.run.smithi086.stdout: daemon grafana.a on smithi192 is in error state 2024-04-02T05:03:51.550 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:51 smithi086 ceph-mon[24811]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T05:03:51.550 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:51 smithi086 ceph-mon[24811]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:03:51.550 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:51 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1299360617' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T05:03:51.550 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:51 smithi086 ceph-mon[29505]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T05:03:51.550 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:51 smithi086 ceph-mon[29505]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:03:51.550 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:51 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1299360617' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T05:03:51.575 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph -s' 2024-04-02T05:03:51.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:51 smithi192 ceph-mon[25697]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T05:03:51.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:51 smithi192 ceph-mon[25697]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:03:51.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:51 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1299360617' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T05:03:53.573 INFO:teuthology.orchestra.run.smithi086.stdout: cluster: 2024-04-02T05:03:53.573 INFO:teuthology.orchestra.run.smithi086.stdout: id: 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 2024-04-02T05:03:53.573 INFO:teuthology.orchestra.run.smithi086.stdout: health: HEALTH_WARN 2024-04-02T05:03:53.573 INFO:teuthology.orchestra.run.smithi086.stdout: 1 failed cephadm daemon(s) 2024-04-02T05:03:53.573 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-02T05:03:53.573 INFO:teuthology.orchestra.run.smithi086.stdout: services: 2024-04-02T05:03:53.573 INFO:teuthology.orchestra.run.smithi086.stdout: mon: 3 daemons, quorum a,c,b (age 18m) 2024-04-02T05:03:53.573 INFO:teuthology.orchestra.run.smithi086.stdout: mgr: x(active, since 6m), standbys: y 2024-04-02T05:03:53.574 INFO:teuthology.orchestra.run.smithi086.stdout: osd: 8 osds: 8 up (since 15m), 8 in (since 15m) 2024-04-02T05:03:53.574 INFO:teuthology.orchestra.run.smithi086.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-04-02T05:03:53.574 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-02T05:03:53.574 INFO:teuthology.orchestra.run.smithi086.stdout: data: 2024-04-02T05:03:53.574 INFO:teuthology.orchestra.run.smithi086.stdout: pools: 6 pools, 161 pgs 2024-04-02T05:03:53.574 INFO:teuthology.orchestra.run.smithi086.stdout: objects: 241 objects, 585 KiB 2024-04-02T05:03:53.574 INFO:teuthology.orchestra.run.smithi086.stdout: usage: 96 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:03:53.574 INFO:teuthology.orchestra.run.smithi086.stdout: pgs: 161 active+clean 2024-04-02T05:03:53.574 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-02T05:03:53.574 INFO:teuthology.orchestra.run.smithi086.stdout: io: 2024-04-02T05:03:53.574 INFO:teuthology.orchestra.run.smithi086.stdout: client: 1.0 KiB/s rd, 1 op/s rd, 0 op/s wr 2024-04-02T05:03:53.574 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-02T05:03:53.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:53 smithi086 ceph-mon[24811]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T05:03:53.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:53 smithi086 ceph-mon[29505]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T05:03:53.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:53 smithi192 ceph-mon[25697]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T05:03:54.146 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph mgr fail' 2024-04-02T05:03:54.709 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:54 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1548993000' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-02T05:03:54.709 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:54 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1548993000' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-02T05:03:54.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:54 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1548993000' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-02T05:03:55.786 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:55 smithi086 ceph-mon[29505]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 585 B/s rd, 0 op/s 2024-04-02T05:03:55.786 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:55 smithi086 ceph-mon[24811]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 585 B/s rd, 0 op/s 2024-04-02T05:03:55.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:55 smithi192 ceph-mon[25697]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 585 B/s rd, 0 op/s 2024-04-02T05:03:57.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:56 smithi192 ceph-mon[25697]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:03:57.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:56 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/3052197088' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-02T05:03:57.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:56 smithi192 ceph-mon[25697]: osdmap e102: 8 total, 8 up, 8 in 2024-04-02T05:03:57.178 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:03:56 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:03:56.843+0000 7f9e87bba700 -1 mgr handle_mgr_map I was active but no longer am 2024-04-02T05:03:57.178 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:03:56 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ignoring --setuser ceph since I am not root 2024-04-02T05:03:57.178 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:03:56 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ignoring --setgroup ceph since I am not root 2024-04-02T05:03:57.178 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:03:56 smithi192 ceph-mgr[45429]: -- 172.21.15.192:0/3004076385 <== mon.2 v2:172.21.15.192:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x558065d905a0 con 0x558065d64800 2024-04-02T05:03:57.187 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:56 smithi086 ceph-mon[29505]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:03:57.187 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:56 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/3052197088' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-02T05:03:57.187 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:56 smithi086 ceph-mon[29505]: osdmap e102: 8 total, 8 up, 8 in 2024-04-02T05:03:57.187 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:03:56 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: [02/Apr/2024:05:03:56] ENGINE Bus STOPPING 2024-04-02T05:03:57.187 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:03:56 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: [02/Apr/2024:05:03:56] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-02T05:03:57.187 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:03:56 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: [02/Apr/2024:05:03:56] ENGINE Bus STOPPED 2024-04-02T05:03:57.187 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:03:56 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: [02/Apr/2024:05:03:56] ENGINE Bus STARTING 2024-04-02T05:03:57.188 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: [02/Apr/2024:05:03:57] ENGINE Serving on http://:::9283 2024-04-02T05:03:57.188 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: [02/Apr/2024:05:03:57] ENGINE Bus STARTED 2024-04-02T05:03:57.188 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:56 smithi086 ceph-mon[24811]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:03:57.188 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:56 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/3052197088' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-02T05:03:57.188 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:56 smithi086 ceph-mon[24811]: osdmap e102: 8 total, 8 up, 8 in 2024-04-02T05:03:57.527 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'sleep 180' 2024-04-02T05:03:57.844 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:03:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:03:57.429+0000 7f712231f200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-02T05:03:57.844 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:03:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:03:57.528+0000 7f712231f200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-02T05:03:58.099 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/3052197088' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-02T05:03:58.099 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[24811]: mgrmap e27: y(active, starting, since 0.572672s) 2024-04-02T05:03:58.099 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T05:03:58.099 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T05:03:58.100 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T05:03:58.100 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T05:03:58.100 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T05:03:58.100 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T05:03:58.100 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T05:03:58.100 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T05:03:58.100 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T05:03:58.100 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T05:03:58.100 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T05:03:58.100 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T05:03:58.100 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T05:03:58.100 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T05:03:58.100 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T05:03:58.100 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[24811]: Manager daemon y is now available 2024-04-02T05:03:58.101 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:03:58.101 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T05:03:58.101 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T05:03:58.101 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/3052197088' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-02T05:03:58.101 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[29505]: mgrmap e27: y(active, starting, since 0.572672s) 2024-04-02T05:03:58.101 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T05:03:58.101 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T05:03:58.101 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T05:03:58.101 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T05:03:58.101 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T05:03:58.101 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T05:03:58.101 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T05:03:58.102 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T05:03:58.102 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T05:03:58.102 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T05:03:58.102 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T05:03:58.102 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T05:03:58.102 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T05:03:58.102 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T05:03:58.102 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T05:03:58.102 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[29505]: Manager daemon y is now available 2024-04-02T05:03:58.102 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:03:58.102 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T05:03:58.102 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:57 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T05:03:58.129 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:57 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/3052197088' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-02T05:03:58.129 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:57 smithi192 ceph-mon[25697]: mgrmap e27: y(active, starting, since 0.572672s) 2024-04-02T05:03:58.130 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:57 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T05:03:58.130 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:57 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T05:03:58.130 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:57 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T05:03:58.130 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:57 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T05:03:58.130 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:57 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T05:03:58.130 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:57 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T05:03:58.130 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:57 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T05:03:58.130 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:57 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T05:03:58.130 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:57 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T05:03:58.130 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:57 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T05:03:58.130 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:57 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T05:03:58.130 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:57 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T05:03:58.130 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:57 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T05:03:58.130 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:57 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T05:03:58.131 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:57 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T05:03:58.131 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:57 smithi192 ceph-mon[25697]: Manager daemon y is now available 2024-04-02T05:03:58.131 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:57 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:03:58.131 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:57 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T05:03:58.131 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:57 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T05:03:58.427 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:03:58 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:03:58.226+0000 7f712231f200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-02T05:03:58.427 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:03:58 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:03:58.319+0000 7f712231f200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-02T05:03:58.755 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:03:58 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:03:58.466+0000 7f712231f200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-02T05:03:58.755 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:03:58 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:03:58.572+0000 7f712231f200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-02T05:03:58.755 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:03:58 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:03:58.655+0000 7f712231f200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-02T05:03:59.005 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:58 smithi192 ceph-mon[25697]: mgrmap e28: y(active, since 1.58068s) 2024-04-02T05:03:59.006 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:03:58 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:03:58.939+0000 7f712231f200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-02T05:03:59.055 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:58 smithi086 ceph-mon[24811]: mgrmap e28: y(active, since 1.58068s) 2024-04-02T05:03:59.055 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:58 smithi086 ceph-mon[29505]: mgrmap e28: y(active, since 1.58068s) 2024-04-02T05:03:59.432 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:03:59 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:03:59.052+0000 7f712231f200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-02T05:03:59.432 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:03:59 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:03:59.140+0000 7f712231f200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-02T05:03:59.432 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:03:59 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:03:59.244+0000 7f712231f200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-02T05:03:59.792 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:03:59 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:03:59.448+0000 7f712231f200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-02T05:04:00.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:59 smithi192 ceph-mon[25697]: [02/Apr/2024:05:03:58] ENGINE Bus STARTING 2024-04-02T05:04:00.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:59 smithi192 ceph-mon[25697]: [02/Apr/2024:05:03:58] ENGINE Serving on http://172.21.15.86:8765 2024-04-02T05:04:00.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:59 smithi192 ceph-mon[25697]: [02/Apr/2024:05:03:58] ENGINE Serving on https://172.21.15.86:7150 2024-04-02T05:04:00.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:59 smithi192 ceph-mon[25697]: [02/Apr/2024:05:03:58] ENGINE Bus STARTED 2024-04-02T05:04:00.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:59 smithi192 ceph-mon[25697]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-02T05:04:00.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:59 smithi192 ceph-mon[25697]: Cluster is now healthy 2024-04-02T05:04:00.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:03:59 smithi192 ceph-mon[25697]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:04:00.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:59 smithi086 ceph-mon[29505]: [02/Apr/2024:05:03:58] ENGINE Bus STARTING 2024-04-02T05:04:00.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:59 smithi086 ceph-mon[29505]: [02/Apr/2024:05:03:58] ENGINE Serving on http://172.21.15.86:8765 2024-04-02T05:04:00.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:59 smithi086 ceph-mon[29505]: [02/Apr/2024:05:03:58] ENGINE Serving on https://172.21.15.86:7150 2024-04-02T05:04:00.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:59 smithi086 ceph-mon[29505]: [02/Apr/2024:05:03:58] ENGINE Bus STARTED 2024-04-02T05:04:00.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:59 smithi086 ceph-mon[29505]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-02T05:04:00.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:59 smithi086 ceph-mon[29505]: Cluster is now healthy 2024-04-02T05:04:00.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:03:59 smithi086 ceph-mon[29505]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:04:00.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:59 smithi086 ceph-mon[24811]: [02/Apr/2024:05:03:58] ENGINE Bus STARTING 2024-04-02T05:04:00.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:59 smithi086 ceph-mon[24811]: [02/Apr/2024:05:03:58] ENGINE Serving on http://172.21.15.86:8765 2024-04-02T05:04:00.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:59 smithi086 ceph-mon[24811]: [02/Apr/2024:05:03:58] ENGINE Serving on https://172.21.15.86:7150 2024-04-02T05:04:00.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:59 smithi086 ceph-mon[24811]: [02/Apr/2024:05:03:58] ENGINE Bus STARTED 2024-04-02T05:04:00.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:59 smithi086 ceph-mon[24811]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-02T05:04:00.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:59 smithi086 ceph-mon[24811]: Cluster is now healthy 2024-04-02T05:04:00.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:03:59 smithi086 ceph-mon[24811]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:04:00.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:04:00 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:04:00.451+0000 7f712231f200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-02T05:04:00.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:04:00 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:04:00.584+0000 7f712231f200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-02T05:04:01.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:00 smithi086 ceph-mon[29505]: mgrmap e29: y(active, since 3s) 2024-04-02T05:04:01.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:00 smithi086 ceph-mon[24811]: mgrmap e29: y(active, since 3s) 2024-04-02T05:04:01.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:00 smithi192 ceph-mon[25697]: mgrmap e29: y(active, since 3s) 2024-04-02T05:04:01.177 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:04:00 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:04:00.937+0000 7f712231f200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-02T05:04:01.178 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:04:01 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:04:01.040+0000 7f712231f200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-02T05:04:01.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:04:01 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:04:01.252+0000 7f712231f200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-02T05:04:01.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:04:01 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:04:01.455+0000 7f712231f200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-02T05:04:02.044 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:04:01 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:04:01.762+0000 7f712231f200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-02T05:04:02.044 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:01 smithi192 ceph-mon[25697]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:04:02.044 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:01 smithi192 ceph-mon[25697]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:04:02.044 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:01 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:02.044 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:01 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:02.135 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:01 smithi086 ceph-mon[24811]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:04:02.135 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:01 smithi086 ceph-mon[24811]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:04:02.135 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:01 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:02.135 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:01 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:02.135 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:01 smithi086 ceph-mon[29505]: from='client.25099 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:04:02.135 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:01 smithi086 ceph-mon[29505]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:04:02.135 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:01 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:02.135 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:01 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:02.299 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:04:02 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:04:02.040+0000 7f712231f200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-02T05:04:02.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:04:02 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:04:02.296+0000 7f712231f200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-02T05:04:03.428 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:04:03 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:04:03.113+0000 7f712231f200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-02T05:04:03.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:03 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:03.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:03 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:03.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:03 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:03.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:03 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:03.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:03 smithi192 ceph-mon[25697]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:04:03.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:03 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:03.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:03 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:03.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:03 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:03.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:03 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:03.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:03 smithi086 ceph-mon[29505]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:04:03.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:03 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:03.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:03 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:03.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:03 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:03.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:03 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:03.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:03 smithi086 ceph-mon[24811]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:04:04.030 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:04:03 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:04:03.690+0000 7f712231f200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-02T05:04:04.030 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:04:03 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:04:03.795+0000 7f712231f200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-02T05:04:04.030 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:04:04 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:04:04.027+0000 7f712231f200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-02T05:04:04.427 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:04:04 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:05:04:04] ENGINE Bus STARTING 2024-04-02T05:04:04.428 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:04:04 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: CherryPy Checker: 2024-04-02T05:04:04.428 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:04:04 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: The Application mounted at '' has an empty config. 2024-04-02T05:04:04.428 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:04:04 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:04:04.428 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:04:04 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:05:04:04] ENGINE Serving on http://:::9283 2024-04-02T05:04:04.428 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:04:04 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:05:04:04] ENGINE Bus STARTED 2024-04-02T05:04:05.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:04 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:05.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:04 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:05.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:04 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:04:05.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:04 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:04:05.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:04 smithi086 ceph-mon[29505]: Updating smithi086:/etc/ceph/ceph.conf 2024-04-02T05:04:05.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:04 smithi086 ceph-mon[29505]: Updating smithi192:/etc/ceph/ceph.conf 2024-04-02T05:04:05.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:04 smithi086 ceph-mon[29505]: Standby manager daemon x started 2024-04-02T05:04:05.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:04 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-02T05:04:05.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:04 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T05:04:05.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:04 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-02T05:04:05.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:04 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T05:04:05.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:04 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:05.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:04 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:05.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:04 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:04:05.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:04 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:04:05.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:04 smithi086 ceph-mon[24811]: Updating smithi086:/etc/ceph/ceph.conf 2024-04-02T05:04:05.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:04 smithi086 ceph-mon[24811]: Updating smithi192:/etc/ceph/ceph.conf 2024-04-02T05:04:05.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:04 smithi086 ceph-mon[24811]: Standby manager daemon x started 2024-04-02T05:04:05.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:04 smithi086 ceph-mon[24811]: from='mgr.? 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-02T05:04:05.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:04 smithi086 ceph-mon[24811]: from='mgr.? 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T05:04:05.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:04 smithi086 ceph-mon[24811]: from='mgr.? 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-02T05:04:05.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:04 smithi086 ceph-mon[24811]: from='mgr.? 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T05:04:05.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:04 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:05.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:04 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:05.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:04 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:04:05.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:04 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:04:05.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:04 smithi192 ceph-mon[25697]: Updating smithi086:/etc/ceph/ceph.conf 2024-04-02T05:04:05.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:04 smithi192 ceph-mon[25697]: Updating smithi192:/etc/ceph/ceph.conf 2024-04-02T05:04:05.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:04 smithi192 ceph-mon[25697]: Standby manager daemon x started 2024-04-02T05:04:05.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:04 smithi192 ceph-mon[25697]: from='mgr.? 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-02T05:04:05.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:04 smithi192 ceph-mon[25697]: from='mgr.? 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T05:04:05.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:04 smithi192 ceph-mon[25697]: from='mgr.? 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-02T05:04:05.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:04 smithi192 ceph-mon[25697]: from='mgr.? 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T05:04:06.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:05 smithi086 ceph-mon[29505]: Updating smithi192:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T05:04:06.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:05 smithi086 ceph-mon[29505]: Updating smithi086:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T05:04:06.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:05 smithi086 ceph-mon[29505]: Updating smithi192:/etc/ceph/ceph.client.admin.keyring 2024-04-02T05:04:06.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:05 smithi086 ceph-mon[29505]: Updating smithi086:/etc/ceph/ceph.client.admin.keyring 2024-04-02T05:04:06.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:05 smithi086 ceph-mon[29505]: Updating smithi086:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-02T05:04:06.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:05 smithi086 ceph-mon[29505]: Updating smithi192:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-02T05:04:06.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:05 smithi086 ceph-mon[29505]: mgrmap e30: y(active, since 8s), standbys: x 2024-04-02T05:04:06.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:05 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T05:04:06.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:05 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:06.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:05 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:06.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:05 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:06.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:05 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:06.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:05 smithi086 ceph-mon[29505]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-02T05:04:06.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:05 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:06.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:05 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:04:06.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:05 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:04:06.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:05 smithi086 ceph-mon[24811]: Updating smithi192:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T05:04:06.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:05 smithi086 ceph-mon[24811]: Updating smithi086:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T05:04:06.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:05 smithi086 ceph-mon[24811]: Updating smithi192:/etc/ceph/ceph.client.admin.keyring 2024-04-02T05:04:06.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:05 smithi086 ceph-mon[24811]: Updating smithi086:/etc/ceph/ceph.client.admin.keyring 2024-04-02T05:04:06.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:05 smithi086 ceph-mon[24811]: Updating smithi086:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-02T05:04:06.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:05 smithi086 ceph-mon[24811]: Updating smithi192:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-02T05:04:06.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:05 smithi086 ceph-mon[24811]: mgrmap e30: y(active, since 8s), standbys: x 2024-04-02T05:04:06.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:05 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T05:04:06.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:05 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:06.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:05 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:06.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:05 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:06.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:05 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:06.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:05 smithi086 ceph-mon[24811]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-02T05:04:06.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:05 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:06.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:05 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:04:06.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:05 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:04:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:05 smithi192 ceph-mon[25697]: Updating smithi192:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T05:04:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:05 smithi192 ceph-mon[25697]: Updating smithi086:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T05:04:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:05 smithi192 ceph-mon[25697]: Updating smithi192:/etc/ceph/ceph.client.admin.keyring 2024-04-02T05:04:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:05 smithi192 ceph-mon[25697]: Updating smithi086:/etc/ceph/ceph.client.admin.keyring 2024-04-02T05:04:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:05 smithi192 ceph-mon[25697]: Updating smithi086:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-02T05:04:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:05 smithi192 ceph-mon[25697]: Updating smithi192:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-02T05:04:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:05 smithi192 ceph-mon[25697]: mgrmap e30: y(active, since 8s), standbys: x 2024-04-02T05:04:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:05 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T05:04:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:05 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:05 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:05 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:06.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:05 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:06.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:05 smithi192 ceph-mon[25697]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-02T05:04:06.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:05 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:06.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:05 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:04:06.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:05 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:04:06.982 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:06 smithi086 ceph-mon[29505]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T05:04:06.982 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:06 smithi086 ceph-mon[29505]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T05:04:06.982 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:06 smithi086 ceph-mon[29505]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-02T05:04:06.982 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:06 smithi086 ceph-mon[24811]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T05:04:06.983 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:06 smithi086 ceph-mon[24811]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T05:04:06.983 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:06 smithi086 ceph-mon[24811]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-02T05:04:07.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:06 smithi192 ceph-mon[25697]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T05:04:07.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:06 smithi192 ceph-mon[25697]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T05:04:07.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:06 smithi192 ceph-mon[25697]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-02T05:04:07.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:04:07 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:04:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T05:04:07.974 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:07 smithi086 ceph-mon[24811]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-02T05:04:07.974 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:07 smithi086 ceph-mon[29505]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-02T05:04:08.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:07 smithi192 ceph-mon[25697]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-02T05:04:09.158 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:08 smithi086 ceph-mon[29505]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-02T05:04:09.158 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:08 smithi086 ceph-mon[24811]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-02T05:04:09.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:08 smithi192 ceph-mon[25697]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-02T05:04:09.677 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:09 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T05:04:09.222Z caller=refresh.go:98 level=error component="discovery manager notify" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.192:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.192:8765: connect: connection refused" 2024-04-02T05:04:09.678 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:09 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T05:04:09.228Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.192:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.192:8765: connect: connection refused" 2024-04-02T05:04:09.678 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:09 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T05:04:09.228Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.192:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.192:8765: connect: connection refused" 2024-04-02T05:04:09.678 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:09 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T05:04:09.228Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.192:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.192:8765: connect: connection refused" 2024-04-02T05:04:10.141 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:10 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:10.141 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:10 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:10.141 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:10 smithi192 ceph-mon[25697]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-02T05:04:10.141 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:10 smithi192 ceph-mon[25697]: Reconfiguring daemon prometheus.a on smithi192 2024-04-02T05:04:10.141 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:10 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1903082397' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T05:04:10.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:10 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:10.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:10 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:10.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:10 smithi086 ceph-mon[24811]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-02T05:04:10.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:10 smithi086 ceph-mon[24811]: Reconfiguring daemon prometheus.a on smithi192 2024-04-02T05:04:10.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:10 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1903082397' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T05:04:10.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:10 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:10.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:10 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:10.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:10 smithi086 ceph-mon[29505]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-02T05:04:10.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:10 smithi086 ceph-mon[29505]: Reconfiguring daemon prometheus.a on smithi192 2024-04-02T05:04:10.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:10 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1903082397' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T05:04:12.166 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:11 smithi192 ceph-mon[25697]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-02T05:04:12.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:11 smithi086 ceph-mon[24811]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-02T05:04:12.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:11 smithi086 ceph-mon[29505]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-02T05:04:12.427 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:12 smithi192 systemd[1]: Stopping Ceph prometheus.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:04:12.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:12 smithi192 bash[54934]: Error: no container with name or ID "ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus.a" found: no such container 2024-04-02T05:04:12.722 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:12 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T05:04:12.451Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-02T05:04:12.722 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:12 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T05:04:12.451Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-02T05:04:12.722 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:12 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T05:04:12.451Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-02T05:04:12.722 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:12 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T05:04:12.451Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-02T05:04:12.722 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:12 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T05:04:12.451Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-02T05:04:12.722 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:12 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T05:04:12.451Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-02T05:04:12.722 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:12 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T05:04:12.451Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-02T05:04:12.722 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:12 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T05:04:12.451Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-02T05:04:12.722 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:12 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T05:04:12.451Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-02T05:04:12.723 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:12 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T05:04:12.451Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-02T05:04:12.723 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:12 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T05:04:12.451Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-02T05:04:12.723 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:12 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[49233]: ts=2024-04-02T05:04:12.451Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-02T05:04:12.723 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:12 smithi192 podman[54947]: 2024-04-02 05:04:12.461441878 +0000 UTC m=+0.112315904 container died d3ec96f3bde3c01aa626ae0c4f0b6f33a8f667fde5e21ecf40eb8b676e862459 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T05:04:13.096 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:12 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:04:13.096 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:12 smithi192 podman[54947]: 2024-04-02 05:04:12.817504842 +0000 UTC m=+0.468378872 container cleanup d3ec96f3bde3c01aa626ae0c4f0b6f33a8f667fde5e21ecf40eb8b676e862459 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T05:04:13.096 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:12 smithi192 bash[54947]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a 2024-04-02T05:04:13.096 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:12 smithi192 bash[54974]: Error: no container with name or ID "ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus.a" found: no such container 2024-04-02T05:04:13.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:12 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:04:13.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:12 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:04:13.355 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:13 smithi192 podman[54961]: 2024-04-02 05:04:13.093765833 +0000 UTC m=+0.634912167 container remove d3ec96f3bde3c01aa626ae0c4f0b6f33a8f667fde5e21ecf40eb8b676e862459 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T05:04:13.355 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:13 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@prometheus.a.service: Deactivated successfully. 2024-04-02T05:04:13.355 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:13 smithi192 systemd[1]: Stopped Ceph prometheus.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:04:13.355 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:13 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@prometheus.a.service: Consumed 1.840s CPU time. 2024-04-02T05:04:13.355 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:13 smithi192 systemd[1]: Starting Ceph prometheus.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:04:13.355 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:13 smithi192 podman[55039]: 2024-04-02 05:04:13.230842395 +0000 UTC m=+0.019134744 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-02T05:04:13.355 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:13 smithi192 podman[55039]: 2024-04-02 05:04:13.353234787 +0000 UTC m=+0.141527112 container create 21cf6d45dead427ae67be089e82e8f5bafde0ee0c88e4007b59d6aad73c6cc6e (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T05:04:13.666 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:13 smithi192 podman[55039]: 2024-04-02 05:04:13.563967454 +0000 UTC m=+0.352259781 container init 21cf6d45dead427ae67be089e82e8f5bafde0ee0c88e4007b59d6aad73c6cc6e (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T05:04:13.666 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:13 smithi192 podman[55039]: 2024-04-02 05:04:13.566529587 +0000 UTC m=+0.354821922 container start 21cf6d45dead427ae67be089e82e8f5bafde0ee0c88e4007b59d6aad73c6cc6e (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T05:04:13.667 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:13 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[55054]: ts=2024-04-02T05:04:13.588Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-02T05:04:13.667 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:13 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[55054]: ts=2024-04-02T05:04:13.588Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-02T05:04:13.667 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:13 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[55054]: ts=2024-04-02T05:04:13.589Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-02T05:04:13.667 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:13 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[55054]: ts=2024-04-02T05:04:13.589Z caller=main.go:518 level=info host_details="(Linux 5.14.0-434.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Mar 26 19:20:47 UTC 2024 x86_64 smithi192 (none))" 2024-04-02T05:04:13.667 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:13 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[55054]: ts=2024-04-02T05:04:13.589Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-02T05:04:13.667 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:13 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[55054]: ts=2024-04-02T05:04:13.589Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-02T05:04:13.667 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:13 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[55054]: ts=2024-04-02T05:04:13.590Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-02T05:04:13.667 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:13 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[55054]: ts=2024-04-02T05:04:13.591Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-02T05:04:13.667 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:13 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[55054]: ts=2024-04-02T05:04:13.592Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-02T05:04:13.667 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:13 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[55054]: ts=2024-04-02T05:04:13.594Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-02T05:04:13.667 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:13 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[55054]: ts=2024-04-02T05:04:13.594Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.72µs 2024-04-02T05:04:13.667 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:13 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[55054]: ts=2024-04-02T05:04:13.594Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-02T05:04:13.667 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:13 smithi192 bash[55039]: 21cf6d45dead427ae67be089e82e8f5bafde0ee0c88e4007b59d6aad73c6cc6e 2024-04-02T05:04:13.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:13 smithi192 ceph-mon[25697]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-02T05:04:13.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:13 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:13.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:13 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:13.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:13 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T05:04:13.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:13 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:13.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:13 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T05:04:13.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:13 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi086"}]: dispatch 2024-04-02T05:04:13.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:13 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:13.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:13 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T05:04:13.929 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:13 smithi192 systemd[1]: Started Ceph prometheus.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:04:13.929 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:13 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[55054]: ts=2024-04-02T05:04:13.796Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-04-02T05:04:14.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:13 smithi086 ceph-mon[29505]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-02T05:04:14.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:13 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:14.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:13 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:14.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:13 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T05:04:14.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:13 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:14.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:13 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T05:04:14.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:13 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi086"}]: dispatch 2024-04-02T05:04:14.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:13 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:14.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:13 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T05:04:14.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:13 smithi086 ceph-mon[24811]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-02T05:04:14.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:13 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:14.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:13 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:14.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:13 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T05:04:14.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:13 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:14.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:13 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T05:04:14.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:13 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi086"}]: dispatch 2024-04-02T05:04:14.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:13 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:14.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:13 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T05:04:14.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[55054]: ts=2024-04-02T05:04:14.020Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-04-02T05:04:14.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[55054]: ts=2024-04-02T05:04:14.081Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-04-02T05:04:14.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[55054]: ts=2024-04-02T05:04:14.081Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-04-02T05:04:14.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[55054]: ts=2024-04-02T05:04:14.081Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=45.96µs wal_replay_duration=487.386699ms total_replay_duration=487.447922ms 2024-04-02T05:04:14.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[55054]: ts=2024-04-02T05:04:14.086Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-02T05:04:14.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[55054]: ts=2024-04-02T05:04:14.086Z caller=main.go:947 level=info msg="TSDB started" 2024-04-02T05:04:14.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[55054]: ts=2024-04-02T05:04:14.086Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-02T05:04:14.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[55054]: ts=2024-04-02T05:04:14.108Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=21.706244ms db_storage=872ns remote_storage=1.697µs web_handler=464ns query_engine=820ns scrape=551.009µs scrape_sd=131.386µs notify=19.012µs notify_sd=16.998µs rules=20.637752ms 2024-04-02T05:04:14.429 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:04:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[55054]: ts=2024-04-02T05:04:14.108Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-02T05:04:15.163 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:14 smithi192 ceph-mon[25697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T05:04:15.164 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:14 smithi192 ceph-mon[25697]: Adding iSCSI gateway http://:@172.21.15.86:5000 to Dashboard 2024-04-02T05:04:15.164 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:14 smithi192 ceph-mon[25697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T05:04:15.164 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:14 smithi192 ceph-mon[25697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi086"}]: dispatch 2024-04-02T05:04:15.164 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:14 smithi192 ceph-mon[25697]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T05:04:15.190 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:14 smithi086 ceph-mon[29505]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T05:04:15.190 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:14 smithi086 ceph-mon[29505]: Adding iSCSI gateway http://:@172.21.15.86:5000 to Dashboard 2024-04-02T05:04:15.190 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:14 smithi086 ceph-mon[29505]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T05:04:15.190 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:14 smithi086 ceph-mon[29505]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi086"}]: dispatch 2024-04-02T05:04:15.190 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:14 smithi086 ceph-mon[29505]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T05:04:15.190 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:14 smithi086 ceph-mon[24811]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T05:04:15.190 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:14 smithi086 ceph-mon[24811]: Adding iSCSI gateway http://:@172.21.15.86:5000 to Dashboard 2024-04-02T05:04:15.190 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:14 smithi086 ceph-mon[24811]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T05:04:15.190 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:14 smithi086 ceph-mon[24811]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi086"}]: dispatch 2024-04-02T05:04:15.191 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:14 smithi086 ceph-mon[24811]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T05:04:16.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:15 smithi192 ceph-mon[25697]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-02T05:04:16.247 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:15 smithi086 ceph-mon[29505]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-02T05:04:16.247 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:15 smithi086 ceph-mon[24811]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-02T05:04:18.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:17 smithi086 ceph-mon[24811]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-02T05:04:18.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:17 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:18.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:17 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:18.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:17 smithi086 ceph-mon[29505]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-02T05:04:18.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:17 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:18.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:17 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:18.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:17 smithi192 ceph-mon[25697]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-02T05:04:18.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:17 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:18.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:17 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:19.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:19 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:19.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:19 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:19.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:19 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:04:19.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:19 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:04:19.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:19 smithi192 ceph-mon[25697]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:04:19.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:19 smithi192 ceph-mon[25697]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 987 B/s rd, 1 op/s 2024-04-02T05:04:19.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:19 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:19.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:19 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:19.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:19 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:19.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:19 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:04:19.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:19 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:04:19.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:19 smithi086 ceph-mon[24811]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:04:19.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:19 smithi086 ceph-mon[24811]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 987 B/s rd, 1 op/s 2024-04-02T05:04:19.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:19 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:19.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:19 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:19.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:19 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:19.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:19 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:04:19.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:19 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:04:19.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:19 smithi086 ceph-mon[29505]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:04:19.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:19 smithi086 ceph-mon[29505]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 987 B/s rd, 1 op/s 2024-04-02T05:04:19.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:19 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:04:20.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:20 smithi192 ceph-mon[25697]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:04:20.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:20 smithi086 ceph-mon[24811]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:04:20.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:20 smithi086 ceph-mon[29505]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:04:22.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:21 smithi192 ceph-mon[25697]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 988 B/s rd, 1 op/s 2024-04-02T05:04:22.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:21 smithi086 ceph-mon[24811]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 988 B/s rd, 1 op/s 2024-04-02T05:04:22.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:21 smithi086 ceph-mon[29505]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 988 B/s rd, 1 op/s 2024-04-02T05:04:23.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:04:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:04:23] "GET /metrics HTTP/1.1" 200 37985 "" "Prometheus/2.33.4" 2024-04-02T05:04:24.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:23 smithi192 ceph-mon[25697]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 988 B/s rd, 1 op/s 2024-04-02T05:04:24.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:23 smithi086 ceph-mon[24811]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 988 B/s rd, 1 op/s 2024-04-02T05:04:24.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:23 smithi086 ceph-mon[29505]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 988 B/s rd, 1 op/s 2024-04-02T05:04:26.117 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:25 smithi192 ceph-mon[25697]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:04:26.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:25 smithi086 ceph-mon[29505]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:04:26.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:25 smithi086 ceph-mon[24811]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:04:28.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:27 smithi192 ceph-mon[25697]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:04:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:27 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:04:28.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:27 smithi086 ceph-mon[24811]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:04:28.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:27 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:04:28.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:27 smithi086 ceph-mon[29505]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:04:28.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:27 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:04:30.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:29 smithi192 ceph-mon[25697]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:04:30.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:29 smithi086 ceph-mon[24811]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:04:30.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:29 smithi086 ceph-mon[29505]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:04:31.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:30 smithi192 ceph-mon[25697]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:04:31.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:30 smithi086 ceph-mon[24811]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:04:31.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:30 smithi086 ceph-mon[29505]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:04:32.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:31 smithi192 ceph-mon[25697]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:04:32.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:31 smithi086 ceph-mon[24811]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:04:32.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:31 smithi086 ceph-mon[29505]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:04:33.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:04:33 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:04:33] "GET /metrics HTTP/1.1" 200 37983 "" "Prometheus/2.33.4" 2024-04-02T05:04:34.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:33 smithi192 ceph-mon[25697]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:04:34.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:33 smithi086 ceph-mon[24811]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:04:34.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:33 smithi086 ceph-mon[29505]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:04:36.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:35 smithi192 ceph-mon[25697]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:04:36.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:35 smithi086 ceph-mon[24811]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:04:36.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:35 smithi086 ceph-mon[29505]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:04:38.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:37 smithi192 ceph-mon[25697]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:04:38.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:37 smithi086 ceph-mon[24811]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:04:38.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:37 smithi086 ceph-mon[29505]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:04:40.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:39 smithi192 ceph-mon[25697]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:04:40.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:39 smithi086 ceph-mon[24811]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:04:40.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:39 smithi086 ceph-mon[29505]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:04:41.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:40 smithi192 ceph-mon[25697]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:04:41.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:40 smithi086 ceph-mon[24811]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:04:41.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:40 smithi086 ceph-mon[29505]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:04:42.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:41 smithi192 ceph-mon[25697]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:04:42.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:41 smithi086 ceph-mon[24811]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:04:42.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:41 smithi086 ceph-mon[29505]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:04:43.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:42 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:04:43.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:42 smithi192 ceph-mon[25697]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:04:43.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:42 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:04:43.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:42 smithi086 ceph-mon[24811]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:04:43.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:42 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:04:43.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:42 smithi086 ceph-mon[29505]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:04:43.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:04:43 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:04:43] "GET /metrics HTTP/1.1" 200 37985 "" "Prometheus/2.33.4" 2024-04-02T05:04:45.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:45 smithi086 ceph-mon[24811]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:04:45.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:45 smithi086 ceph-mon[29505]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:04:45.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:45 smithi192 ceph-mon[25697]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:04:48.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:47 smithi192 ceph-mon[25697]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:04:48.287 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:47 smithi086 ceph-mon[24811]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:04:48.287 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:47 smithi086 ceph-mon[29505]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:04:50.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:49 smithi192 ceph-mon[25697]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:04:50.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:49 smithi086 ceph-mon[24811]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:04:50.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:49 smithi086 ceph-mon[29505]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:04:51.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:50 smithi192 ceph-mon[25697]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:04:51.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:50 smithi086 ceph-mon[24811]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:04:51.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:50 smithi086 ceph-mon[29505]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:04:52.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:51 smithi192 ceph-mon[25697]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:04:52.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:51 smithi086 ceph-mon[29505]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:04:52.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:51 smithi086 ceph-mon[24811]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:04:53.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:04:53 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:04:53] "GET /metrics HTTP/1.1" 200 37985 "" "Prometheus/2.33.4" 2024-04-02T05:04:54.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:53 smithi192 ceph-mon[25697]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:04:54.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:53 smithi086 ceph-mon[29505]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:04:54.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:53 smithi086 ceph-mon[24811]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:04:56.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:55 smithi192 ceph-mon[25697]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:04:56.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:55 smithi086 ceph-mon[24811]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:04:56.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:55 smithi086 ceph-mon[29505]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:04:58.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:57 smithi192 ceph-mon[25697]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:04:58.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:57 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:04:58.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:57 smithi086 ceph-mon[24811]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:04:58.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:57 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:04:58.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:57 smithi086 ceph-mon[29505]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:04:58.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:57 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:05:00.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:04:59 smithi192 ceph-mon[25697]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:05:00.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:04:59 smithi086 ceph-mon[29505]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:05:00.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:04:59 smithi086 ceph-mon[24811]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:05:01.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:00 smithi192 ceph-mon[25697]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:05:01.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:00 smithi086 ceph-mon[24811]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:05:01.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:00 smithi086 ceph-mon[29505]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:05:02.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:01 smithi192 ceph-mon[25697]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:02.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:01 smithi086 ceph-mon[24811]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:02.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:01 smithi086 ceph-mon[29505]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:03.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:02 smithi086 ceph-mon[24811]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:03.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:02 smithi086 ceph-mon[29505]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:03.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:02 smithi192 ceph-mon[25697]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:03.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:05:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:05:03] "GET /metrics HTTP/1.1" 200 37977 "" "Prometheus/2.33.4" 2024-04-02T05:05:05.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:05 smithi086 ceph-mon[24811]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:05:05.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:05 smithi086 ceph-mon[29505]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:05:05.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:05 smithi192 ceph-mon[25697]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:05:08.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:07 smithi192 ceph-mon[25697]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:08.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:07 smithi086 ceph-mon[24811]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:08.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:07 smithi086 ceph-mon[29505]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:10.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:09 smithi192 ceph-mon[25697]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:05:10.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:09 smithi086 ceph-mon[24811]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:05:10.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:09 smithi086 ceph-mon[29505]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:05:11.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:10 smithi192 ceph-mon[25697]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:05:11.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:10 smithi086 ceph-mon[24811]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:05:11.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:10 smithi086 ceph-mon[29505]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:05:12.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:11 smithi192 ceph-mon[25697]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:12.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:11 smithi086 ceph-mon[24811]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:12.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:11 smithi086 ceph-mon[29505]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:13.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:12 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:05:13.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:12 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:05:13.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:12 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:05:13.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:05:13 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:05:13] "GET /metrics HTTP/1.1" 200 37980 "" "Prometheus/2.33.4" 2024-04-02T05:05:14.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:13 smithi192 ceph-mon[25697]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:14.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:13 smithi086 ceph-mon[24811]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:14.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:13 smithi086 ceph-mon[29505]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:16.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:15 smithi192 ceph-mon[25697]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:05:16.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:15 smithi086 ceph-mon[29505]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:05:16.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:15 smithi086 ceph-mon[24811]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:05:18.135 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:17 smithi086 ceph-mon[29505]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:18.135 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:17 smithi086 ceph-mon[24811]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:18.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:17 smithi192 ceph-mon[25697]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:20.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:19 smithi086 ceph-mon[24811]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:05:20.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:19 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:05:20.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:19 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:05:20.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:19 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:05:20.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:19 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:05:20.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:19 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:05:20.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:19 smithi086 ceph-mon[29505]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:05:20.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:19 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:05:20.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:19 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:05:20.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:19 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:05:20.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:19 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:05:20.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:19 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:05:20.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:19 smithi192 ceph-mon[25697]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:05:20.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:19 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:05:20.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:19 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:05:20.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:19 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:05:20.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:19 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:05:20.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:19 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:05:21.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:20 smithi192 ceph-mon[25697]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:05:21.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:20 smithi192 ceph-mon[25697]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 912 B/s rd, 0 op/s 2024-04-02T05:05:21.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:20 smithi192 ceph-mon[25697]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:05:21.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:20 smithi086 ceph-mon[24811]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:05:21.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:20 smithi086 ceph-mon[24811]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 912 B/s rd, 0 op/s 2024-04-02T05:05:21.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:20 smithi086 ceph-mon[24811]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:05:21.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:20 smithi086 ceph-mon[29505]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:05:21.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:20 smithi086 ceph-mon[29505]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 912 B/s rd, 0 op/s 2024-04-02T05:05:21.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:20 smithi086 ceph-mon[29505]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:05:23.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:22 smithi192 ceph-mon[25697]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:05:23.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:22 smithi086 ceph-mon[24811]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:05:23.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:22 smithi086 ceph-mon[29505]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:05:23.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:05:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:05:23] "GET /metrics HTTP/1.1" 200 37980 "" "Prometheus/2.33.4" 2024-04-02T05:05:25.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:24 smithi192 ceph-mon[25697]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:05:25.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:24 smithi086 ceph-mon[24811]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:05:25.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:24 smithi086 ceph-mon[29505]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:05:27.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:26 smithi192 ceph-mon[25697]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:05:27.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:26 smithi086 ceph-mon[24811]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:05:27.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:26 smithi086 ceph-mon[29505]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:05:28.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:27 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:05:28.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:27 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:05:28.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:27 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:05:29.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:28 smithi192 ceph-mon[25697]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 555 B/s rd, 0 op/s 2024-04-02T05:05:29.230 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:28 smithi086 ceph-mon[29505]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 555 B/s rd, 0 op/s 2024-04-02T05:05:29.230 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:28 smithi086 ceph-mon[24811]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 555 B/s rd, 0 op/s 2024-04-02T05:05:31.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:30 smithi192 ceph-mon[25697]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:05:31.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:30 smithi192 ceph-mon[25697]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:05:31.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:30 smithi086 ceph-mon[24811]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:05:31.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:30 smithi086 ceph-mon[24811]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:05:31.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:30 smithi086 ceph-mon[29505]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:05:31.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:30 smithi086 ceph-mon[29505]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:05:33.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:32 smithi192 ceph-mon[25697]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:33.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:32 smithi086 ceph-mon[29505]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:33.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:32 smithi086 ceph-mon[24811]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:33.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:05:33 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:05:33] "GET /metrics HTTP/1.1" 200 37981 "" "Prometheus/2.33.4" 2024-04-02T05:05:35.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:34 smithi192 ceph-mon[25697]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:05:35.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:34 smithi086 ceph-mon[24811]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:05:35.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:34 smithi086 ceph-mon[29505]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:05:37.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:36 smithi192 ceph-mon[25697]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:37.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:36 smithi086 ceph-mon[29505]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:37.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:36 smithi086 ceph-mon[24811]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:39.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:38 smithi192 ceph-mon[25697]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:39.239 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:38 smithi086 ceph-mon[29505]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:39.239 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:38 smithi086 ceph-mon[24811]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:41.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:40 smithi192 ceph-mon[25697]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:05:41.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:40 smithi192 ceph-mon[25697]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:05:41.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:40 smithi086 ceph-mon[24811]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:05:41.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:40 smithi086 ceph-mon[24811]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:05:41.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:40 smithi086 ceph-mon[29505]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:05:41.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:40 smithi086 ceph-mon[29505]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:05:42.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:41 smithi086 ceph-mon[24811]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:42.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:41 smithi086 ceph-mon[29505]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:42.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:41 smithi192 ceph-mon[25697]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:43.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:42 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:05:43.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:42 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:05:43.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:42 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:05:43.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:05:43 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:05:43] "GET /metrics HTTP/1.1" 200 37983 "" "Prometheus/2.33.4" 2024-04-02T05:05:44.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:43 smithi086 ceph-mon[29505]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:05:44.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:43 smithi086 ceph-mon[24811]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:05:44.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:43 smithi192 ceph-mon[25697]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:05:47.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:46 smithi192 ceph-mon[25697]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:47.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:46 smithi086 ceph-mon[29505]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:47.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:46 smithi086 ceph-mon[24811]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:49.135 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:48 smithi086 ceph-mon[29505]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:49.135 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:48 smithi086 ceph-mon[24811]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:49.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:48 smithi192 ceph-mon[25697]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:51.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:50 smithi192 ceph-mon[25697]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:05:51.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:50 smithi192 ceph-mon[25697]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:05:51.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:50 smithi086 ceph-mon[29505]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:05:51.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:50 smithi086 ceph-mon[29505]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:05:51.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:50 smithi086 ceph-mon[24811]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:05:51.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:50 smithi086 ceph-mon[24811]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:05:53.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:52 smithi192 ceph-mon[25697]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:53.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:52 smithi086 ceph-mon[24811]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:53.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:52 smithi086 ceph-mon[29505]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:53.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:05:53 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:05:53] "GET /metrics HTTP/1.1" 200 37983 "" "Prometheus/2.33.4" 2024-04-02T05:05:55.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:54 smithi192 ceph-mon[25697]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:05:55.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:54 smithi086 ceph-mon[24811]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:05:55.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:54 smithi086 ceph-mon[29505]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:05:57.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:56 smithi192 ceph-mon[25697]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:57.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:56 smithi086 ceph-mon[24811]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:57.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:56 smithi086 ceph-mon[29505]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:58.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:57 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:05:58.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:57 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:05:58.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:57 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:05:59.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:05:58 smithi192 ceph-mon[25697]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:59.256 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:05:58 smithi086 ceph-mon[24811]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:05:59.256 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:05:58 smithi086 ceph-mon[29505]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:01.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:00 smithi192 ceph-mon[25697]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:06:01.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:00 smithi192 ceph-mon[25697]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:06:01.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:00 smithi086 ceph-mon[24811]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:06:01.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:00 smithi086 ceph-mon[24811]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:06:01.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:00 smithi086 ceph-mon[29505]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:06:01.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:00 smithi086 ceph-mon[29505]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:06:03.181 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:02 smithi192 ceph-mon[25697]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:03.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:02 smithi086 ceph-mon[24811]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:03.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:02 smithi086 ceph-mon[29505]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:03.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:06:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:06:03] "GET /metrics HTTP/1.1" 200 37982 "" "Prometheus/2.33.4" 2024-04-02T05:06:05.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:04 smithi192 ceph-mon[25697]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:06:05.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:04 smithi086 ceph-mon[29505]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:06:05.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:04 smithi086 ceph-mon[24811]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:06:07.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:06 smithi192 ceph-mon[25697]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:07.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:06 smithi086 ceph-mon[24811]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:07.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:06 smithi086 ceph-mon[29505]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:09.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:08 smithi192 ceph-mon[25697]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:09.265 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:08 smithi086 ceph-mon[24811]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:09.265 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:08 smithi086 ceph-mon[29505]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:11.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:10 smithi192 ceph-mon[25697]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:06:11.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:10 smithi192 ceph-mon[25697]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:06:11.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:10 smithi086 ceph-mon[24811]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:06:11.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:10 smithi086 ceph-mon[24811]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:06:11.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:10 smithi086 ceph-mon[29505]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:06:11.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:10 smithi086 ceph-mon[29505]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:06:13.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:12 smithi192 ceph-mon[25697]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:13.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:12 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:06:13.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:12 smithi086 ceph-mon[24811]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:13.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:12 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:06:13.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:12 smithi086 ceph-mon[29505]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:13.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:12 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:06:13.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:06:13 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:06:13] "GET /metrics HTTP/1.1" 200 37992 "" "Prometheus/2.33.4" 2024-04-02T05:06:15.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:14 smithi192 ceph-mon[25697]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:06:15.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:14 smithi086 ceph-mon[24811]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:06:15.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:14 smithi086 ceph-mon[29505]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:06:17.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:16 smithi192 ceph-mon[25697]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:17.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:16 smithi086 ceph-mon[24811]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:17.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:16 smithi086 ceph-mon[29505]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:19.135 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:18 smithi086 ceph-mon[29505]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:19.135 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:18 smithi086 ceph-mon[24811]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:19.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:18 smithi192 ceph-mon[25697]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:21.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:20 smithi192 ceph-mon[25697]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:06:21.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:20 smithi192 ceph-mon[25697]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:06:21.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:20 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:06:21.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:20 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:06:21.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:20 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:06:21.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:20 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:06:21.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:20 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:06:21.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:20 smithi086 ceph-mon[24811]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:06:21.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:20 smithi086 ceph-mon[24811]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:06:21.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:20 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:06:21.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:20 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:06:21.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:20 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:06:21.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:20 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:06:21.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:20 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:06:21.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:20 smithi086 ceph-mon[29505]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:06:21.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:20 smithi086 ceph-mon[29505]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:06:21.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:20 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:06:21.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:20 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:06:21.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:20 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:06:21.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:20 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:06:21.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:20 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' 2024-04-02T05:06:22.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:21 smithi192 ceph-mon[25697]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 913 B/s rd, 0 op/s 2024-04-02T05:06:22.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:21 smithi192 ceph-mon[25697]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:06:22.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:21 smithi086 ceph-mon[24811]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 913 B/s rd, 0 op/s 2024-04-02T05:06:22.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:21 smithi086 ceph-mon[24811]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:06:22.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:21 smithi086 ceph-mon[29505]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 913 B/s rd, 0 op/s 2024-04-02T05:06:22.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:21 smithi086 ceph-mon[29505]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:06:23.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:06:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:06:23] "GET /metrics HTTP/1.1" 200 37992 "" "Prometheus/2.33.4" 2024-04-02T05:06:24.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:23 smithi192 ceph-mon[25697]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 555 B/s rd, 0 op/s 2024-04-02T05:06:24.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:23 smithi086 ceph-mon[29505]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 555 B/s rd, 0 op/s 2024-04-02T05:06:24.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:23 smithi086 ceph-mon[24811]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 555 B/s rd, 0 op/s 2024-04-02T05:06:26.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:25 smithi192 ceph-mon[25697]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:06:26.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:25 smithi086 ceph-mon[24811]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:06:26.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:25 smithi086 ceph-mon[29505]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:06:28.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:27 smithi192 ceph-mon[25697]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:06:28.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:27 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:06:28.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:27 smithi086 ceph-mon[29505]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:06:28.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:27 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:06:28.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:27 smithi086 ceph-mon[24811]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:06:28.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:27 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:06:30.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:29 smithi192 ceph-mon[25697]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:06:30.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:29 smithi086 ceph-mon[29505]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:06:30.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:29 smithi086 ceph-mon[24811]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:06:31.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:30 smithi192 ceph-mon[25697]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:06:31.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:30 smithi192 ceph-mon[25697]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:06:31.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:30 smithi086 ceph-mon[24811]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:06:31.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:30 smithi086 ceph-mon[24811]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:06:31.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:30 smithi086 ceph-mon[29505]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:06:31.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:30 smithi086 ceph-mon[29505]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:06:33.670 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:06:33 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:06:33] "GET /metrics HTTP/1.1" 200 37992 "" "Prometheus/2.33.4" 2024-04-02T05:06:33.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:33 smithi192 ceph-mon[25697]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:34.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:33 smithi086 ceph-mon[24811]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:34.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:33 smithi086 ceph-mon[29505]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:35.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:35 smithi192 ceph-mon[25697]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:06:36.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:35 smithi086 ceph-mon[24811]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:06:36.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:35 smithi086 ceph-mon[29505]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:06:38.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:37 smithi192 ceph-mon[25697]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:38.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:37 smithi086 ceph-mon[24811]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:38.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:37 smithi086 ceph-mon[29505]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:40.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:39 smithi192 ceph-mon[25697]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:06:40.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:39 smithi086 ceph-mon[29505]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:06:40.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:39 smithi086 ceph-mon[24811]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:06:41.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:40 smithi192 ceph-mon[25697]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:06:41.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:40 smithi086 ceph-mon[29505]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:06:41.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:40 smithi086 ceph-mon[24811]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:06:42.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:41 smithi192 ceph-mon[25697]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:42.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:41 smithi086 ceph-mon[29505]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:42.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:41 smithi086 ceph-mon[24811]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:43.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:42 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:06:43.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:42 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:06:43.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:42 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:06:43.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:06:43 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:06:43] "GET /metrics HTTP/1.1" 200 37993 "" "Prometheus/2.33.4" 2024-04-02T05:06:44.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:43 smithi192 ceph-mon[25697]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:44.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:43 smithi086 ceph-mon[24811]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:44.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:43 smithi086 ceph-mon[29505]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:45.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:44 smithi086 ceph-mon[24811]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:06:45.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:44 smithi086 ceph-mon[29505]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:06:45.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:44 smithi192 ceph-mon[25697]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:06:48.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:47 smithi192 ceph-mon[25697]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:48.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:47 smithi086 ceph-mon[24811]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:48.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:47 smithi086 ceph-mon[29505]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:50.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:49 smithi192 ceph-mon[25697]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:06:50.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:49 smithi086 ceph-mon[24811]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:06:50.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:49 smithi086 ceph-mon[29505]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:06:51.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:50 smithi192 ceph-mon[25697]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:06:51.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:50 smithi086 ceph-mon[24811]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:06:51.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:50 smithi086 ceph-mon[29505]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:06:52.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:51 smithi192 ceph-mon[25697]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:52.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:51 smithi086 ceph-mon[24811]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:52.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:51 smithi086 ceph-mon[29505]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:53.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:06:53 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:06:53] "GET /metrics HTTP/1.1" 200 37993 "" "Prometheus/2.33.4" 2024-04-02T05:06:54.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:53 smithi192 ceph-mon[25697]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:54.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:53 smithi086 ceph-mon[29505]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:54.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:53 smithi086 ceph-mon[24811]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:56.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:55 smithi192 ceph-mon[25697]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:06:56.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:55 smithi086 ceph-mon[24811]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:06:56.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:55 smithi086 ceph-mon[29505]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:06:58.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:57 smithi192 ceph-mon[25697]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:58.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:57 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:06:58.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:57 smithi086 ceph-mon[24811]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:58.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:57 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:06:58.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:57 smithi086 ceph-mon[29505]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:06:58.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:57 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:06:59.639 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch ps' 2024-04-02T05:07:00.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:06:59 smithi192 ceph-mon[25697]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:07:00.181 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:06:59 smithi086 ceph-mon[29505]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:07:00.181 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:06:59 smithi086 ceph-mon[24811]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:07:00.935 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:00 smithi086 ceph-mon[24811]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:07:00.935 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:00 smithi086 ceph-mon[29505]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:07:01.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:00 smithi192 ceph-mon[25697]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:07:01.620 INFO:teuthology.orchestra.run.smithi086.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T05:07:01.620 INFO:teuthology.orchestra.run.smithi086.stdout:alertmanager.a smithi086 *:9093,9094 running (9m) 2m ago 17m 22.6M - 0.23.0 ba2b418f427c b2769c8e287a 2024-04-02T05:07:01.620 INFO:teuthology.orchestra.run.smithi086.stdout:grafana.a smithi192 *:3000 error 2m ago 16m - - 2024-04-02T05:07:01.620 INFO:teuthology.orchestra.run.smithi086.stdout:iscsi.foo.smithi086.rsbqwx smithi086 running (2m) 2m ago 16m 48.1M - 3.5 e1d6a67b021e 977de5f79b25 2024-04-02T05:07:01.621 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.x smithi192 *:8443,9283 running (12m) 2m ago 21m 447M - 19.0.0-1979-g6cec793d 78f830190839 fd618149d6b4 2024-04-02T05:07:01.621 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.y smithi086 *:8443,9283,8765 running (5m) 2m ago 23m 498M - 19.0.0-1979-g6cec793d 78f830190839 3dca54a018ab 2024-04-02T05:07:01.621 INFO:teuthology.orchestra.run.smithi086.stdout:mon.a smithi086 running (23m) 2m ago 23m 65.0M 2048M 17.2.0 e1d6a67b021e b6c507ab96f8 2024-04-02T05:07:01.621 INFO:teuthology.orchestra.run.smithi086.stdout:mon.b smithi192 running (21m) 2m ago 21m 47.5M 2048M 17.2.0 e1d6a67b021e 4dadd8d1c318 2024-04-02T05:07:01.621 INFO:teuthology.orchestra.run.smithi086.stdout:mon.c smithi086 running (21m) 2m ago 21m 44.3M 2048M 17.2.0 e1d6a67b021e 49275483a2d4 2024-04-02T05:07:01.621 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.a smithi086 *:9100 running (9m) 2m ago 17m 18.0M - 1.3.1 1dbe0e931976 843a5e9887fb 2024-04-02T05:07:01.621 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.b smithi192 *:9100 running (8m) 2m ago 17m 17.0M - 1.3.1 1dbe0e931976 bd849d6c4fb0 2024-04-02T05:07:01.621 INFO:teuthology.orchestra.run.smithi086.stdout:osd.0 smithi086 running (21m) 2m ago 21m 56.2M 2992M 17.2.0 e1d6a67b021e 6d90c60d1c71 2024-04-02T05:07:01.621 INFO:teuthology.orchestra.run.smithi086.stdout:osd.1 smithi086 running (20m) 2m ago 20m 54.6M 2992M 17.2.0 e1d6a67b021e 203689f71603 2024-04-02T05:07:01.621 INFO:teuthology.orchestra.run.smithi086.stdout:osd.2 smithi086 running (20m) 2m ago 20m 49.5M 2992M 17.2.0 e1d6a67b021e 472c860a9af8 2024-04-02T05:07:01.622 INFO:teuthology.orchestra.run.smithi086.stdout:osd.3 smithi086 running (19m) 2m ago 19m 52.0M 2992M 17.2.0 e1d6a67b021e 327538dac44e 2024-04-02T05:07:01.622 INFO:teuthology.orchestra.run.smithi086.stdout:osd.4 smithi192 running (19m) 2m ago 19m 53.2M 3248M 17.2.0 e1d6a67b021e 6535d7919a2d 2024-04-02T05:07:01.622 INFO:teuthology.orchestra.run.smithi086.stdout:osd.5 smithi192 running (19m) 2m ago 19m 53.4M 3248M 17.2.0 e1d6a67b021e 7259249605f2 2024-04-02T05:07:01.622 INFO:teuthology.orchestra.run.smithi086.stdout:osd.6 smithi192 running (18m) 2m ago 18m 50.6M 3248M 17.2.0 e1d6a67b021e bb7d171b02a1 2024-04-02T05:07:01.622 INFO:teuthology.orchestra.run.smithi086.stdout:osd.7 smithi192 running (18m) 2m ago 18m 53.4M 3248M 17.2.0 e1d6a67b021e 95f5a0353999 2024-04-02T05:07:01.622 INFO:teuthology.orchestra.run.smithi086.stdout:prometheus.a smithi192 *:9095 running (2m) 2m ago 17m 44.8M - 2.33.4 514e6a882f6e 21cf6d45dead 2024-04-02T05:07:01.622 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi086.bxzxlp smithi086 *:8000 running (16m) 2m ago 16m 76.0M - 17.2.0 e1d6a67b021e 6725785fbc53 2024-04-02T05:07:01.622 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi192.pmlbot smithi192 *:8000 running (16m) 2m ago 16m 75.2M - 17.2.0 e1d6a67b021e 5ae3638f8b1c 2024-04-02T05:07:02.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:01 smithi192 ceph-mon[25697]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:07:02.208 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:01 smithi086 ceph-mon[29505]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:07:02.208 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:01 smithi086 ceph-mon[24811]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:07:02.233 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph versions' 2024-04-02T05:07:03.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:02 smithi086 ceph-mon[24811]: from='client.25192 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:07:03.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:02 smithi086 ceph-mon[24811]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:07:03.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:02 smithi086 ceph-mon[29505]: from='client.25192 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:07:03.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:02 smithi086 ceph-mon[29505]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:07:03.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:02 smithi192 ceph-mon[25697]: from='client.25192 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:07:03.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:02 smithi192 ceph-mon[25697]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:07:03.738 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:07:03] "GET /metrics HTTP/1.1" 200 37988 "" "Prometheus/2.33.4" 2024-04-02T05:07:04.239 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:07:04.239 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": { 2024-04-02T05:07:04.239 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-02T05:07:04.240 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:07:04.240 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": { 2024-04-02T05:07:04.240 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 2 2024-04-02T05:07:04.240 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:07:04.240 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": { 2024-04-02T05:07:04.240 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-02T05:07:04.240 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:07:04.240 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": {}, 2024-04-02T05:07:04.240 INFO:teuthology.orchestra.run.smithi086.stdout: "rgw": { 2024-04-02T05:07:04.240 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-02T05:07:04.240 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:07:04.240 INFO:teuthology.orchestra.run.smithi086.stdout: "overall": { 2024-04-02T05:07:04.240 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-02T05:07:04.240 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 2 2024-04-02T05:07:04.241 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-04-02T05:07:04.241 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:07:04.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:04 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/3450256889' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:07:04.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:04 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/3450256889' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:07:04.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:04 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/3450256889' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:07:04.937 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph -s' 2024-04-02T05:07:05.574 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:05 smithi086 ceph-mon[29505]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:07:05.574 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:05 smithi086 ceph-mon[24811]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:07:05.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:05 smithi192 ceph-mon[25697]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:07:06.997 INFO:teuthology.orchestra.run.smithi086.stdout: cluster: 2024-04-02T05:07:06.998 INFO:teuthology.orchestra.run.smithi086.stdout: id: 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 2024-04-02T05:07:06.998 INFO:teuthology.orchestra.run.smithi086.stdout: health: HEALTH_WARN 2024-04-02T05:07:06.998 INFO:teuthology.orchestra.run.smithi086.stdout: 1 failed cephadm daemon(s) 2024-04-02T05:07:06.998 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-02T05:07:06.998 INFO:teuthology.orchestra.run.smithi086.stdout: services: 2024-04-02T05:07:06.998 INFO:teuthology.orchestra.run.smithi086.stdout: mon: 3 daemons, quorum a,c,b (age 21m) 2024-04-02T05:07:06.998 INFO:teuthology.orchestra.run.smithi086.stdout: mgr: y(active, since 3m), standbys: x 2024-04-02T05:07:06.998 INFO:teuthology.orchestra.run.smithi086.stdout: osd: 8 osds: 8 up (since 18m), 8 in (since 18m) 2024-04-02T05:07:06.998 INFO:teuthology.orchestra.run.smithi086.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-04-02T05:07:06.998 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-02T05:07:06.998 INFO:teuthology.orchestra.run.smithi086.stdout: data: 2024-04-02T05:07:06.998 INFO:teuthology.orchestra.run.smithi086.stdout: pools: 6 pools, 161 pgs 2024-04-02T05:07:06.998 INFO:teuthology.orchestra.run.smithi086.stdout: objects: 241 objects, 585 KiB 2024-04-02T05:07:06.998 INFO:teuthology.orchestra.run.smithi086.stdout: usage: 96 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:07:06.999 INFO:teuthology.orchestra.run.smithi086.stdout: pgs: 161 active+clean 2024-04-02T05:07:06.999 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-02T05:07:06.999 INFO:teuthology.orchestra.run.smithi086.stdout: io: 2024-04-02T05:07:06.999 INFO:teuthology.orchestra.run.smithi086.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-04-02T05:07:06.999 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-02T05:07:07.574 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph health detail' 2024-04-02T05:07:08.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:07 smithi192 ceph-mon[25697]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:07:08.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:07 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/3438679689' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-02T05:07:08.212 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:07 smithi086 ceph-mon[24811]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:07:08.213 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:07 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/3438679689' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-02T05:07:08.213 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:07 smithi086 ceph-mon[29505]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:07:08.213 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:07 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/3438679689' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-02T05:07:09.407 INFO:teuthology.orchestra.run.smithi086.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-02T05:07:09.407 INFO:teuthology.orchestra.run.smithi086.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-02T05:07:09.407 INFO:teuthology.orchestra.run.smithi086.stdout: daemon grafana.a on smithi192 is in error state 2024-04-02T05:07:09.989 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:09 smithi086 ceph-mon[24811]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:07:09.989 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:09 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/3581950015' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T05:07:09.989 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:09 smithi086 ceph-mon[29505]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:07:09.989 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:09 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/3581950015' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T05:07:10.011 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-04-02T05:07:10.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:09 smithi192 ceph-mon[25697]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:07:10.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:09 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/3581950015' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T05:07:11.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:10 smithi192 ceph-mon[25697]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:07:11.195 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:10 smithi086 ceph-mon[24811]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:07:11.195 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:10 smithi086 ceph-mon[29505]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:07:11.988 INFO:teuthology.orchestra.run.smithi086.stdout:true 2024-04-02T05:07:12.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:11 smithi192 ceph-mon[25697]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:07:12.252 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:11 smithi086 ceph-mon[29505]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:07:12.253 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:11 smithi086 ceph-mon[24811]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:07:12.574 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph mgr fail' 2024-04-02T05:07:13.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:12 smithi192 ceph-mon[25697]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:07:13.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:12 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1663026743' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:07:13.214 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:12 smithi086 ceph-mon[29505]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:07:13.214 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:12 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1663026743' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:07:13.214 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:12 smithi086 ceph-mon[24811]: from='mgr.15189 172.21.15.86:0/3316112941' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:07:13.214 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:12 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1663026743' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:07:13.471 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:13 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:07:13] "GET /metrics HTTP/1.1" 200 37990 "" "Prometheus/2.33.4" 2024-04-02T05:07:14.107 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:13 smithi086 ceph-mon[29505]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:07:14.107 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:13 smithi086 ceph-mon[24811]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:07:14.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:13 smithi192 ceph-mon[25697]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:07:15.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:14 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/4256940407' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-02T05:07:15.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:14 smithi192 ceph-mon[25697]: osdmap e103: 8 total, 8 up, 8 in 2024-04-02T05:07:15.177 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:07:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:05:07:14] ENGINE Bus STOPPING 2024-04-02T05:07:15.177 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:07:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:05:07:14] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-02T05:07:15.178 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:07:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:05:07:14] ENGINE Bus STOPPED 2024-04-02T05:07:15.178 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:07:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:05:07:14] ENGINE Bus STARTING 2024-04-02T05:07:15.178 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:07:15 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:05:07:15] ENGINE Serving on http://:::9283 2024-04-02T05:07:15.178 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:07:15 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:05:07:15] ENGINE Bus STARTED 2024-04-02T05:07:15.232 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:14 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/4256940407' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-02T05:07:15.233 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:14 smithi086 ceph-mon[24811]: osdmap e103: 8 total, 8 up, 8 in 2024-04-02T05:07:15.233 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:14 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/4256940407' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-02T05:07:15.233 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:14 smithi086 ceph-mon[29505]: osdmap e103: 8 total, 8 up, 8 in 2024-04-02T05:07:15.233 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:14 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ignoring --setuser ceph since I am not root 2024-04-02T05:07:15.233 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:14 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ignoring --setgroup ceph since I am not root 2024-04-02T05:07:15.233 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:14 smithi086 ceph-mgr[62645]: -- 172.21.15.86:0/2957151869 <== mon.1 v2:172.21.15.86:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x557f50d865a0 con 0x557f50d5ac00 2024-04-02T05:07:15.517 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'sleep 180' 2024-04-02T05:07:15.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:07:15.492+0000 7f188206a200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-02T05:07:15.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:07:15.596+0000 7f188206a200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-02T05:07:16.148 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[29505]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:07:16.149 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/4256940407' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-02T05:07:16.149 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[29505]: mgrmap e31: x(active, starting, since 0.323049s) 2024-04-02T05:07:16.149 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T05:07:16.149 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T05:07:16.149 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T05:07:16.149 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T05:07:16.149 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T05:07:16.149 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T05:07:16.149 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T05:07:16.149 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T05:07:16.149 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T05:07:16.149 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T05:07:16.149 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T05:07:16.149 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T05:07:16.150 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T05:07:16.150 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T05:07:16.150 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T05:07:16.150 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[29505]: Manager daemon x is now available 2024-04-02T05:07:16.150 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:07:16.150 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-02T05:07:16.150 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[29505]: from='mgr.15255 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-02T05:07:16.150 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-02T05:07:16.150 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[29505]: from='mgr.15255 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-02T05:07:16.150 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[24811]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:07:16.150 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/4256940407' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-02T05:07:16.150 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[24811]: mgrmap e31: x(active, starting, since 0.323049s) 2024-04-02T05:07:16.151 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T05:07:16.151 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T05:07:16.151 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T05:07:16.151 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T05:07:16.151 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T05:07:16.151 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T05:07:16.151 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T05:07:16.151 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T05:07:16.151 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T05:07:16.151 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T05:07:16.151 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T05:07:16.151 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T05:07:16.151 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T05:07:16.151 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T05:07:16.151 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T05:07:16.151 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[24811]: Manager daemon x is now available 2024-04-02T05:07:16.152 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:07:16.152 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-02T05:07:16.152 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[24811]: from='mgr.15255 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-02T05:07:16.152 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-02T05:07:16.152 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:15 smithi086 ceph-mon[24811]: from='mgr.15255 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-02T05:07:16.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:15 smithi192 ceph-mon[25697]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:07:16.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:15 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/4256940407' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-02T05:07:16.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:15 smithi192 ceph-mon[25697]: mgrmap e31: x(active, starting, since 0.323049s) 2024-04-02T05:07:16.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:15 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T05:07:16.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:15 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T05:07:16.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:15 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T05:07:16.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:15 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T05:07:16.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:15 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T05:07:16.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:15 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T05:07:16.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:15 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T05:07:16.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:15 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T05:07:16.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:15 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T05:07:16.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:15 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T05:07:16.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:15 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T05:07:16.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:15 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T05:07:16.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:15 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T05:07:16.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:15 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T05:07:16.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:15 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T05:07:16.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:15 smithi192 ceph-mon[25697]: Manager daemon x is now available 2024-04-02T05:07:16.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:15 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:07:16.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:15 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-02T05:07:16.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:15 smithi192 ceph-mon[25697]: from='mgr.15255 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-02T05:07:16.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:15 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-02T05:07:16.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:15 smithi192 ceph-mon[25697]: from='mgr.15255 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-02T05:07:16.502 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:16 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:07:16.284+0000 7f188206a200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-02T05:07:16.502 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:16 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:07:16.387+0000 7f188206a200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-02T05:07:16.793 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:16 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:07:16.537+0000 7f188206a200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-02T05:07:16.793 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:16 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:07:16.634+0000 7f188206a200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-02T05:07:16.793 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:16 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:07:16.721+0000 7f188206a200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-02T05:07:17.074 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:17 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:07:17.002+0000 7f188206a200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-02T05:07:17.074 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:16 smithi086 ceph-mon[29505]: mgrmap e32: x(active, since 1.32563s) 2024-04-02T05:07:17.074 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:16 smithi086 ceph-mon[29505]: [02/Apr/2024:05:07:15] ENGINE Bus STARTING 2024-04-02T05:07:17.074 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:16 smithi086 ceph-mon[29505]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:07:17.074 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:16 smithi086 ceph-mon[24811]: mgrmap e32: x(active, since 1.32563s) 2024-04-02T05:07:17.075 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:16 smithi086 ceph-mon[24811]: [02/Apr/2024:05:07:15] ENGINE Bus STARTING 2024-04-02T05:07:17.075 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:16 smithi086 ceph-mon[24811]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:07:17.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:16 smithi192 ceph-mon[25697]: mgrmap e32: x(active, since 1.32563s) 2024-04-02T05:07:17.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:16 smithi192 ceph-mon[25697]: [02/Apr/2024:05:07:15] ENGINE Bus STARTING 2024-04-02T05:07:17.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:16 smithi192 ceph-mon[25697]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:07:17.333 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:17 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:07:17.122+0000 7f188206a200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-02T05:07:17.333 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:17 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:07:17.212+0000 7f188206a200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-02T05:07:17.333 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:17 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:07:17.315+0000 7f188206a200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-02T05:07:17.691 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:17 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:07:17.500+0000 7f188206a200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-02T05:07:18.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:17 smithi086 ceph-mon[24811]: [02/Apr/2024:05:07:16] ENGINE Serving on http://172.21.15.192:8765 2024-04-02T05:07:18.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:17 smithi086 ceph-mon[24811]: [02/Apr/2024:05:07:16] ENGINE Serving on https://172.21.15.192:7150 2024-04-02T05:07:18.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:17 smithi086 ceph-mon[24811]: [02/Apr/2024:05:07:16] ENGINE Bus STARTED 2024-04-02T05:07:18.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:17 smithi086 ceph-mon[24811]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-02T05:07:18.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:17 smithi086 ceph-mon[24811]: Cluster is now healthy 2024-04-02T05:07:18.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:17 smithi086 ceph-mon[24811]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:07:18.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:17 smithi086 ceph-mon[24811]: mgrmap e33: x(active, since 2s) 2024-04-02T05:07:18.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:17 smithi086 ceph-mon[29505]: [02/Apr/2024:05:07:16] ENGINE Serving on http://172.21.15.192:8765 2024-04-02T05:07:18.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:17 smithi086 ceph-mon[29505]: [02/Apr/2024:05:07:16] ENGINE Serving on https://172.21.15.192:7150 2024-04-02T05:07:18.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:17 smithi086 ceph-mon[29505]: [02/Apr/2024:05:07:16] ENGINE Bus STARTED 2024-04-02T05:07:18.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:17 smithi086 ceph-mon[29505]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-02T05:07:18.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:17 smithi086 ceph-mon[29505]: Cluster is now healthy 2024-04-02T05:07:18.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:17 smithi086 ceph-mon[29505]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:07:18.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:17 smithi086 ceph-mon[29505]: mgrmap e33: x(active, since 2s) 2024-04-02T05:07:18.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:17 smithi192 ceph-mon[25697]: [02/Apr/2024:05:07:16] ENGINE Serving on http://172.21.15.192:8765 2024-04-02T05:07:18.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:17 smithi192 ceph-mon[25697]: [02/Apr/2024:05:07:16] ENGINE Serving on https://172.21.15.192:7150 2024-04-02T05:07:18.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:17 smithi192 ceph-mon[25697]: [02/Apr/2024:05:07:16] ENGINE Bus STARTED 2024-04-02T05:07:18.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:17 smithi192 ceph-mon[25697]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-02T05:07:18.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:17 smithi192 ceph-mon[25697]: Cluster is now healthy 2024-04-02T05:07:18.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:17 smithi192 ceph-mon[25697]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:07:18.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:17 smithi192 ceph-mon[25697]: mgrmap e33: x(active, since 2s) 2024-04-02T05:07:18.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:18 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:07:18.498+0000 7f188206a200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-02T05:07:18.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:18 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:07:18.640+0000 7f188206a200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-02T05:07:19.263 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:19 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:07:19.007+0000 7f188206a200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-02T05:07:19.264 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:19 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:07:19.103+0000 7f188206a200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-02T05:07:19.529 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:19 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:07:19.307+0000 7f188206a200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-02T05:07:19.832 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:19 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:07:19.525+0000 7f188206a200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-02T05:07:20.126 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:19 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:07:19.846+0000 7f188206a200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-02T05:07:20.126 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:19 smithi086 ceph-mon[24811]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:07:20.126 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:19 smithi086 ceph-mon[24811]: mgrmap e34: x(active, since 4s) 2024-04-02T05:07:20.126 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:19 smithi086 ceph-mon[24811]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:20.126 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:19 smithi086 ceph-mon[24811]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:20.127 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:19 smithi086 ceph-mon[29505]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:07:20.127 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:19 smithi086 ceph-mon[29505]: mgrmap e34: x(active, since 4s) 2024-04-02T05:07:20.127 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:19 smithi086 ceph-mon[29505]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:20.127 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:19 smithi086 ceph-mon[29505]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:20.384 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:07:20.122+0000 7f188206a200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-02T05:07:20.385 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:20 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:07:20.381+0000 7f188206a200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-02T05:07:20.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:19 smithi192 ceph-mon[25697]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:07:20.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:19 smithi192 ceph-mon[25697]: mgrmap e34: x(active, since 4s) 2024-04-02T05:07:20.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:19 smithi192 ceph-mon[25697]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:20.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:19 smithi192 ceph-mon[25697]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:21.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:20 smithi086 ceph-mon[24811]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:07:21.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:20 smithi086 ceph-mon[24811]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:21.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:20 smithi086 ceph-mon[24811]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:21.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:20 smithi086 ceph-mon[29505]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:07:21.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:20 smithi086 ceph-mon[29505]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:21.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:20 smithi086 ceph-mon[29505]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:21.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:20 smithi192 ceph-mon[25697]: from='client.34931 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:07:21.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:20 smithi192 ceph-mon[25697]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:21.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:20 smithi192 ceph-mon[25697]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:21.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:21 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:07:21.253+0000 7f188206a200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-02T05:07:22.200 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:21 smithi086 ceph-mon[29505]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:07:22.201 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:21 smithi086 ceph-mon[29505]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:22.201 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:21 smithi086 ceph-mon[29505]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:22.201 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:21 smithi086 ceph-mon[24811]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:07:22.201 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:21 smithi086 ceph-mon[24811]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:22.201 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:21 smithi086 ceph-mon[24811]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:22.201 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:21 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:07:21.853+0000 7f188206a200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-02T05:07:22.201 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:21 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:07:21.960+0000 7f188206a200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-02T05:07:22.202 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:22 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:07:22.196+0000 7f188206a200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-02T05:07:22.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:21 smithi192 ceph-mon[25697]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:07:22.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:21 smithi192 ceph-mon[25697]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:22.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:21 smithi192 ceph-mon[25697]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:22.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:22 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: [02/Apr/2024:05:07:22] ENGINE Bus STARTING 2024-04-02T05:07:22.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:22 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: CherryPy Checker: 2024-04-02T05:07:22.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:22 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: The Application mounted at '' has an empty config. 2024-04-02T05:07:22.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:22 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:07:22.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:22 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: [02/Apr/2024:05:07:22] ENGINE Serving on http://:::9283 2024-04-02T05:07:22.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:22 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: [02/Apr/2024:05:07:22] ENGINE Bus STARTED 2024-04-02T05:07:23.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:23 smithi192 ceph-mon[25697]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:23.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:23 smithi192 ceph-mon[25697]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:23.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:23 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:07:23.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:23 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:07:23.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:23 smithi192 ceph-mon[25697]: Updating smithi086:/etc/ceph/ceph.conf 2024-04-02T05:07:23.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:23 smithi192 ceph-mon[25697]: Updating smithi192:/etc/ceph/ceph.conf 2024-04-02T05:07:23.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:23 smithi192 ceph-mon[25697]: Standby manager daemon y started 2024-04-02T05:07:23.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:23 smithi192 ceph-mon[25697]: from='mgr.? 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-02T05:07:23.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:23 smithi192 ceph-mon[25697]: from='mgr.? 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T05:07:23.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:23 smithi192 ceph-mon[25697]: from='mgr.? 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-02T05:07:23.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:23 smithi192 ceph-mon[25697]: from='mgr.? 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T05:07:23.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:23 smithi192 ceph-mon[25697]: Updating smithi086:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T05:07:23.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:23 smithi192 ceph-mon[25697]: Updating smithi192:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T05:07:23.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:23 smithi192 ceph-mon[25697]: Updating smithi086:/etc/ceph/ceph.client.admin.keyring 2024-04-02T05:07:23.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:23 smithi192 ceph-mon[25697]: Updating smithi192:/etc/ceph/ceph.client.admin.keyring 2024-04-02T05:07:23.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:23 smithi192 ceph-mon[25697]: Updating smithi086:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-02T05:07:23.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:23 smithi192 ceph-mon[25697]: Updating smithi192:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-02T05:07:23.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:23 smithi192 ceph-mon[25697]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-02T05:07:23.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:23 smithi192 ceph-mon[25697]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:23.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:23 smithi192 ceph-mon[25697]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:23.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:23 smithi192 ceph-mon[25697]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:23.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:23 smithi192 ceph-mon[25697]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:23.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:23 smithi192 ceph-mon[25697]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:23.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:07:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T05:07:23.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[29505]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:23.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[29505]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:23.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:07:23.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:07:23.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[29505]: Updating smithi086:/etc/ceph/ceph.conf 2024-04-02T05:07:23.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[29505]: Updating smithi192:/etc/ceph/ceph.conf 2024-04-02T05:07:23.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[29505]: Standby manager daemon y started 2024-04-02T05:07:23.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-02T05:07:23.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T05:07:23.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-02T05:07:23.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T05:07:23.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[29505]: Updating smithi086:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T05:07:23.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[29505]: Updating smithi192:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T05:07:23.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[29505]: Updating smithi086:/etc/ceph/ceph.client.admin.keyring 2024-04-02T05:07:23.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[29505]: Updating smithi192:/etc/ceph/ceph.client.admin.keyring 2024-04-02T05:07:23.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[29505]: Updating smithi086:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-02T05:07:23.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[29505]: Updating smithi192:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-02T05:07:23.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[29505]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-02T05:07:23.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[29505]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:23.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[29505]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:23.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[29505]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:23.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[29505]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:23.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[29505]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:23.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[24811]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:23.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[24811]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:23.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:07:23.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:07:23.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[24811]: Updating smithi086:/etc/ceph/ceph.conf 2024-04-02T05:07:23.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[24811]: Updating smithi192:/etc/ceph/ceph.conf 2024-04-02T05:07:23.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[24811]: Standby manager daemon y started 2024-04-02T05:07:23.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[24811]: from='mgr.? 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-02T05:07:23.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[24811]: from='mgr.? 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T05:07:23.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[24811]: from='mgr.? 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-02T05:07:23.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[24811]: from='mgr.? 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T05:07:23.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[24811]: Updating smithi086:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T05:07:23.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[24811]: Updating smithi192:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T05:07:23.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[24811]: Updating smithi086:/etc/ceph/ceph.client.admin.keyring 2024-04-02T05:07:23.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[24811]: Updating smithi192:/etc/ceph/ceph.client.admin.keyring 2024-04-02T05:07:23.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[24811]: Updating smithi086:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-02T05:07:23.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[24811]: Updating smithi192:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-02T05:07:23.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[24811]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-02T05:07:23.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[24811]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:23.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[24811]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:23.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[24811]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:23.587 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[24811]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:23.587 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:23 smithi086 ceph-mon[24811]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:24.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:24 smithi086 ceph-mon[29505]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-02T05:07:24.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:24 smithi086 ceph-mon[29505]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T05:07:24.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:24 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:07:24.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:24 smithi086 ceph-mon[29505]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-02T05:07:24.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:24 smithi086 ceph-mon[29505]: from='mgr.15255 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:07:24.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:24 smithi086 ceph-mon[29505]: mgrmap e35: x(active, since 8s), standbys: y 2024-04-02T05:07:24.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:24 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T05:07:24.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:24 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:07:24.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:24 smithi086 ceph-mon[29505]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T05:07:24.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:24 smithi086 ceph-mon[24811]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-02T05:07:24.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:24 smithi086 ceph-mon[24811]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T05:07:24.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:24 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:07:24.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:24 smithi086 ceph-mon[24811]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-02T05:07:24.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:24 smithi086 ceph-mon[24811]: from='mgr.15255 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:07:24.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:24 smithi086 ceph-mon[24811]: mgrmap e35: x(active, since 8s), standbys: y 2024-04-02T05:07:24.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:24 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T05:07:24.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:24 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:07:24.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:24 smithi086 ceph-mon[24811]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T05:07:24.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:24 smithi192 ceph-mon[25697]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-02T05:07:24.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:24 smithi192 ceph-mon[25697]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T05:07:24.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:24 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:07:24.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:24 smithi192 ceph-mon[25697]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-02T05:07:24.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:24 smithi192 ceph-mon[25697]: from='mgr.15255 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:07:24.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:24 smithi192 ceph-mon[25697]: mgrmap e35: x(active, since 8s), standbys: y 2024-04-02T05:07:24.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:24 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T05:07:24.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:24 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:07:24.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:24 smithi192 ceph-mon[25697]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T05:07:27.066 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:26 smithi086 ceph-mon[24811]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-02T05:07:27.067 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:26 smithi086 ceph-mon[29505]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-02T05:07:27.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:26 smithi192 ceph-mon[25697]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-02T05:07:28.534 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:28 smithi192 ceph-mon[25697]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-02T05:07:28.534 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:28 smithi192 ceph-mon[25697]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:28.534 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:28 smithi192 ceph-mon[25697]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:28.534 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:28 smithi192 ceph-mon[25697]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-02T05:07:28.534 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:28 smithi192 ceph-mon[25697]: Reconfiguring daemon prometheus.a on smithi192 2024-04-02T05:07:28.534 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:28 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1471433161' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T05:07:28.534 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:28 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/2700253441' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2224887480"}]: dispatch 2024-04-02T05:07:28.534 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:28 smithi192 ceph-mon[25697]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2224887480"}]: dispatch 2024-04-02T05:07:28.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:28 smithi086 ceph-mon[24811]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-02T05:07:28.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:28 smithi086 ceph-mon[24811]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:28.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:28 smithi086 ceph-mon[24811]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:28.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:28 smithi086 ceph-mon[24811]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-02T05:07:28.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:28 smithi086 ceph-mon[24811]: Reconfiguring daemon prometheus.a on smithi192 2024-04-02T05:07:28.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:28 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1471433161' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T05:07:28.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:28 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/2700253441' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2224887480"}]: dispatch 2024-04-02T05:07:28.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:28 smithi086 ceph-mon[24811]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2224887480"}]: dispatch 2024-04-02T05:07:28.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:28 smithi086 ceph-mon[29505]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-02T05:07:28.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:28 smithi086 ceph-mon[29505]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:28.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:28 smithi086 ceph-mon[29505]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:28.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:28 smithi086 ceph-mon[29505]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-02T05:07:28.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:28 smithi086 ceph-mon[29505]: Reconfiguring daemon prometheus.a on smithi192 2024-04-02T05:07:28.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:28 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1471433161' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T05:07:28.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:28 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/2700253441' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2224887480"}]: dispatch 2024-04-02T05:07:28.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:28 smithi086 ceph-mon[29505]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2224887480"}]: dispatch 2024-04-02T05:07:30.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:29 smithi192 ceph-mon[25697]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2224887480"}]': finished 2024-04-02T05:07:30.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:29 smithi192 ceph-mon[25697]: osdmap e104: 8 total, 8 up, 8 in 2024-04-02T05:07:30.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:29 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1329901037' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1523799922"}]: dispatch 2024-04-02T05:07:30.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:29 smithi192 ceph-mon[25697]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1523799922"}]: dispatch 2024-04-02T05:07:30.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:29 smithi086 ceph-mon[24811]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2224887480"}]': finished 2024-04-02T05:07:30.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:29 smithi086 ceph-mon[24811]: osdmap e104: 8 total, 8 up, 8 in 2024-04-02T05:07:30.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:29 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1329901037' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1523799922"}]: dispatch 2024-04-02T05:07:30.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:29 smithi086 ceph-mon[24811]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1523799922"}]: dispatch 2024-04-02T05:07:30.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:29 smithi086 ceph-mon[29505]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2224887480"}]': finished 2024-04-02T05:07:30.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:29 smithi086 ceph-mon[29505]: osdmap e104: 8 total, 8 up, 8 in 2024-04-02T05:07:30.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:29 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1329901037' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1523799922"}]: dispatch 2024-04-02T05:07:30.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:29 smithi086 ceph-mon[29505]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1523799922"}]: dispatch 2024-04-02T05:07:31.015 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:30 smithi192 systemd[1]: Stopping Ceph prometheus.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:07:31.015 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:30 smithi192 bash[57904]: Error: no container with name or ID "ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus.a" found: no such container 2024-04-02T05:07:31.015 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:30 smithi192 ceph-mon[25697]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 15 op/s 2024-04-02T05:07:31.015 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:30 smithi192 ceph-mon[25697]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1523799922"}]': finished 2024-04-02T05:07:31.015 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:30 smithi192 ceph-mon[25697]: osdmap e105: 8 total, 8 up, 8 in 2024-04-02T05:07:31.015 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:30 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:07:31.015 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:30 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/3784464204' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1561636961"}]: dispatch 2024-04-02T05:07:31.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:30 smithi086 ceph-mon[24811]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 15 op/s 2024-04-02T05:07:31.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:30 smithi086 ceph-mon[24811]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1523799922"}]': finished 2024-04-02T05:07:31.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:30 smithi086 ceph-mon[24811]: osdmap e105: 8 total, 8 up, 8 in 2024-04-02T05:07:31.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:30 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:07:31.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:30 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/3784464204' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1561636961"}]: dispatch 2024-04-02T05:07:31.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:30 smithi086 ceph-mon[29505]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 15 op/s 2024-04-02T05:07:31.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:30 smithi086 ceph-mon[29505]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1523799922"}]': finished 2024-04-02T05:07:31.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:30 smithi086 ceph-mon[29505]: osdmap e105: 8 total, 8 up, 8 in 2024-04-02T05:07:31.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:30 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:07:31.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:30 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/3784464204' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1561636961"}]: dispatch 2024-04-02T05:07:31.371 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:31 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[55054]: ts=2024-04-02T05:07:31.012Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-02T05:07:31.371 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:31 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[55054]: ts=2024-04-02T05:07:31.012Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-02T05:07:31.371 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:31 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[55054]: ts=2024-04-02T05:07:31.012Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-02T05:07:31.371 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:31 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[55054]: ts=2024-04-02T05:07:31.012Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-02T05:07:31.371 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:31 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[55054]: ts=2024-04-02T05:07:31.012Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-02T05:07:31.371 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:31 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[55054]: ts=2024-04-02T05:07:31.012Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-02T05:07:31.371 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:31 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[55054]: ts=2024-04-02T05:07:31.012Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-02T05:07:31.371 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:31 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[55054]: ts=2024-04-02T05:07:31.012Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-02T05:07:31.371 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:31 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[55054]: ts=2024-04-02T05:07:31.012Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-02T05:07:31.372 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:31 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[55054]: ts=2024-04-02T05:07:31.013Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-02T05:07:31.372 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:31 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[55054]: ts=2024-04-02T05:07:31.013Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-02T05:07:31.372 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:31 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[55054]: ts=2024-04-02T05:07:31.013Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-02T05:07:31.372 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:31 smithi192 podman[57915]: 2024-04-02 05:07:31.022544707 +0000 UTC m=+0.143285793 container died 21cf6d45dead427ae67be089e82e8f5bafde0ee0c88e4007b59d6aad73c6cc6e (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T05:07:31.649 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:31 smithi192 podman[57915]: 2024-04-02 05:07:31.390048669 +0000 UTC m=+0.510789735 container cleanup 21cf6d45dead427ae67be089e82e8f5bafde0ee0c88e4007b59d6aad73c6cc6e (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T05:07:31.649 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:31 smithi192 bash[57915]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a 2024-04-02T05:07:31.649 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:31 smithi192 bash[57942]: Error: no container with name or ID "ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus.a" found: no such container 2024-04-02T05:07:31.908 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:31 smithi192 podman[57929]: 2024-04-02 05:07:31.646920506 +0000 UTC m=+0.627115886 container remove 21cf6d45dead427ae67be089e82e8f5bafde0ee0c88e4007b59d6aad73c6cc6e (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T05:07:31.908 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:31 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@prometheus.a.service: Deactivated successfully. 2024-04-02T05:07:31.908 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:31 smithi192 systemd[1]: Stopped Ceph prometheus.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:07:31.908 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:31 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@prometheus.a.service: Consumed 1.296s CPU time. 2024-04-02T05:07:31.908 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:31 smithi192 systemd[1]: Starting Ceph prometheus.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:07:31.908 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:31 smithi192 podman[58014]: 2024-04-02 05:07:31.784274266 +0000 UTC m=+0.018997612 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-02T05:07:31.908 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:31 smithi192 podman[58014]: 2024-04-02 05:07:31.906057195 +0000 UTC m=+0.140780522 container create f455b2541990855818e04134d01876b2840e7ee552a43cb652163b1ac3d40d8b (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T05:07:32.159 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:31 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/3784464204' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1561636961"}]': finished 2024-04-02T05:07:32.159 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:31 smithi192 ceph-mon[25697]: osdmap e106: 8 total, 8 up, 8 in 2024-04-02T05:07:32.159 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:31 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/2718768115' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/371189065"}]: dispatch 2024-04-02T05:07:32.159 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:32 smithi192 podman[58014]: 2024-04-02 05:07:32.12498639 +0000 UTC m=+0.359709725 container init f455b2541990855818e04134d01876b2840e7ee552a43cb652163b1ac3d40d8b (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T05:07:32.159 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:32 smithi192 podman[58014]: 2024-04-02 05:07:32.127449006 +0000 UTC m=+0.362172334 container start f455b2541990855818e04134d01876b2840e7ee552a43cb652163b1ac3d40d8b (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T05:07:32.159 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:07:32.150Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-02T05:07:32.159 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:07:32.150Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-02T05:07:32.159 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:07:32.150Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-02T05:07:32.159 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:07:32.150Z caller=main.go:518 level=info host_details="(Linux 5.14.0-434.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Mar 26 19:20:47 UTC 2024 x86_64 smithi192 (none))" 2024-04-02T05:07:32.159 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:07:32.150Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-02T05:07:32.160 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:07:32.151Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-02T05:07:32.160 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:07:32.152Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-02T05:07:32.160 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:07:32.153Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-02T05:07:32.160 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:07:32.153Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-02T05:07:32.160 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:07:32.156Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-02T05:07:32.160 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:07:32.156Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.723µs 2024-04-02T05:07:32.160 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:07:32.156Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-02T05:07:32.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:31 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/3784464204' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1561636961"}]': finished 2024-04-02T05:07:32.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:31 smithi086 ceph-mon[29505]: osdmap e106: 8 total, 8 up, 8 in 2024-04-02T05:07:32.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:31 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/2718768115' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/371189065"}]: dispatch 2024-04-02T05:07:32.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:31 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/3784464204' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1561636961"}]': finished 2024-04-02T05:07:32.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:31 smithi086 ceph-mon[24811]: osdmap e106: 8 total, 8 up, 8 in 2024-04-02T05:07:32.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:31 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/2718768115' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/371189065"}]: dispatch 2024-04-02T05:07:32.427 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:32 smithi192 bash[58014]: f455b2541990855818e04134d01876b2840e7ee552a43cb652163b1ac3d40d8b 2024-04-02T05:07:32.427 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:32 smithi192 systemd[1]: Started Ceph prometheus.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:07:32.427 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:07:32.346Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=4 2024-04-02T05:07:32.911 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:07:32.585Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=4 2024-04-02T05:07:32.911 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:07:32.647Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=4 2024-04-02T05:07:32.911 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:07:32.698Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=4 2024-04-02T05:07:32.911 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:07:32.698Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=4 2024-04-02T05:07:32.911 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:07:32.698Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=39.706µs wal_replay_duration=542.236334ms total_replay_duration=542.286717ms 2024-04-02T05:07:32.911 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:07:32.704Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-02T05:07:32.911 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:07:32.704Z caller=main.go:947 level=info msg="TSDB started" 2024-04-02T05:07:32.911 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:07:32.704Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-02T05:07:32.911 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:07:32.721Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=17.036278ms db_storage=711ns remote_storage=1.4µs web_handler=372ns query_engine=644ns scrape=367.708µs scrape_sd=96.082µs notify=15.254µs notify_sd=16.7µs rules=16.327341ms 2024-04-02T05:07:32.911 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:07:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:07:32.721Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-02T05:07:33.173 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:32 smithi192 ceph-mon[25697]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-04-02T05:07:33.173 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:32 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/2718768115' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/371189065"}]': finished 2024-04-02T05:07:33.173 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:32 smithi192 ceph-mon[25697]: osdmap e107: 8 total, 8 up, 8 in 2024-04-02T05:07:33.173 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:32 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1955831679' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/371189065"}]: dispatch 2024-04-02T05:07:33.173 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:32 smithi192 ceph-mon[25697]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/371189065"}]: dispatch 2024-04-02T05:07:33.174 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:32 smithi192 ceph-mon[25697]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:33.174 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:32 smithi192 ceph-mon[25697]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:33.174 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:32 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T05:07:33.174 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:32 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T05:07:33.174 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:32 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T05:07:33.303 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:32 smithi086 ceph-mon[29505]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-04-02T05:07:33.303 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:32 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/2718768115' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/371189065"}]': finished 2024-04-02T05:07:33.303 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:32 smithi086 ceph-mon[29505]: osdmap e107: 8 total, 8 up, 8 in 2024-04-02T05:07:33.303 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:32 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1955831679' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/371189065"}]: dispatch 2024-04-02T05:07:33.303 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:32 smithi086 ceph-mon[29505]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/371189065"}]: dispatch 2024-04-02T05:07:33.303 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:32 smithi086 ceph-mon[29505]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:33.304 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:32 smithi086 ceph-mon[29505]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:33.304 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:32 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T05:07:33.304 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:32 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T05:07:33.304 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:32 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T05:07:33.304 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:32 smithi086 ceph-mon[24811]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-04-02T05:07:33.304 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:32 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/2718768115' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/371189065"}]': finished 2024-04-02T05:07:33.304 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:32 smithi086 ceph-mon[24811]: osdmap e107: 8 total, 8 up, 8 in 2024-04-02T05:07:33.304 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:32 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1955831679' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/371189065"}]: dispatch 2024-04-02T05:07:33.304 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:32 smithi086 ceph-mon[24811]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/371189065"}]: dispatch 2024-04-02T05:07:33.304 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:32 smithi086 ceph-mon[24811]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:33.304 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:32 smithi086 ceph-mon[24811]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:33.304 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:32 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T05:07:33.305 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:32 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T05:07:33.305 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:32 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T05:07:34.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:33 smithi086 ceph-mon[24811]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T05:07:34.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:33 smithi086 ceph-mon[24811]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T05:07:34.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:33 smithi086 ceph-mon[24811]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T05:07:34.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:33 smithi086 ceph-mon[24811]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/371189065"}]': finished 2024-04-02T05:07:34.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:33 smithi086 ceph-mon[24811]: osdmap e108: 8 total, 8 up, 8 in 2024-04-02T05:07:34.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:33 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1743324630' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2101528088"}]: dispatch 2024-04-02T05:07:34.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:33 smithi086 ceph-mon[24811]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2101528088"}]: dispatch 2024-04-02T05:07:34.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:33 smithi086 ceph-mon[29505]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T05:07:34.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:33 smithi086 ceph-mon[29505]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T05:07:34.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:33 smithi086 ceph-mon[29505]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T05:07:34.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:33 smithi086 ceph-mon[29505]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/371189065"}]': finished 2024-04-02T05:07:34.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:33 smithi086 ceph-mon[29505]: osdmap e108: 8 total, 8 up, 8 in 2024-04-02T05:07:34.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:33 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1743324630' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2101528088"}]: dispatch 2024-04-02T05:07:34.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:33 smithi086 ceph-mon[29505]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2101528088"}]: dispatch 2024-04-02T05:07:34.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:33 smithi192 ceph-mon[25697]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T05:07:34.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:33 smithi192 ceph-mon[25697]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T05:07:34.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:33 smithi192 ceph-mon[25697]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T05:07:34.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:33 smithi192 ceph-mon[25697]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/371189065"}]': finished 2024-04-02T05:07:34.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:33 smithi192 ceph-mon[25697]: osdmap e108: 8 total, 8 up, 8 in 2024-04-02T05:07:34.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:33 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1743324630' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2101528088"}]: dispatch 2024-04-02T05:07:34.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:33 smithi192 ceph-mon[25697]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2101528088"}]: dispatch 2024-04-02T05:07:35.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:34 smithi086 ceph-mon[29505]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-04-02T05:07:35.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:34 smithi086 ceph-mon[29505]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2101528088"}]': finished 2024-04-02T05:07:35.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:34 smithi086 ceph-mon[29505]: osdmap e109: 8 total, 8 up, 8 in 2024-04-02T05:07:35.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:34 smithi086 ceph-mon[24811]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-04-02T05:07:35.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:34 smithi086 ceph-mon[24811]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2101528088"}]': finished 2024-04-02T05:07:35.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:34 smithi086 ceph-mon[24811]: osdmap e109: 8 total, 8 up, 8 in 2024-04-02T05:07:35.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:34 smithi192 ceph-mon[25697]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-04-02T05:07:35.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:34 smithi192 ceph-mon[25697]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/2101528088"}]': finished 2024-04-02T05:07:35.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:34 smithi192 ceph-mon[25697]: osdmap e109: 8 total, 8 up, 8 in 2024-04-02T05:07:37.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:36 smithi192 ceph-mon[25697]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 972 B/s rd, 1 op/s 2024-04-02T05:07:37.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:36 smithi192 ceph-mon[25697]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:37.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:36 smithi192 ceph-mon[25697]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:37.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:36 smithi192 ceph-mon[25697]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:37.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:36 smithi192 ceph-mon[25697]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:37.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:36 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:07:37.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:36 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:07:37.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:36 smithi192 ceph-mon[25697]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:37.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:36 smithi086 ceph-mon[29505]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 972 B/s rd, 1 op/s 2024-04-02T05:07:37.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:36 smithi086 ceph-mon[29505]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:37.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:36 smithi086 ceph-mon[29505]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:37.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:36 smithi086 ceph-mon[29505]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:37.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:36 smithi086 ceph-mon[29505]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:37.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:36 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:07:37.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:36 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:07:37.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:36 smithi086 ceph-mon[29505]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:37.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:36 smithi086 ceph-mon[24811]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 972 B/s rd, 1 op/s 2024-04-02T05:07:37.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:36 smithi086 ceph-mon[24811]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:37.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:36 smithi086 ceph-mon[24811]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:37.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:36 smithi086 ceph-mon[24811]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:37.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:36 smithi086 ceph-mon[24811]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:37.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:36 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:07:37.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:36 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:07:37.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:36 smithi086 ceph-mon[24811]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:07:38.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:37 smithi086 ceph-mon[29505]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 927 B/s rd, 0 op/s 2024-04-02T05:07:38.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:37 smithi086 ceph-mon[29505]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:07:38.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:37 smithi086 ceph-mon[24811]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 927 B/s rd, 0 op/s 2024-04-02T05:07:38.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:37 smithi086 ceph-mon[24811]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:07:38.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:37 smithi192 ceph-mon[25697]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 927 B/s rd, 0 op/s 2024-04-02T05:07:38.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:37 smithi192 ceph-mon[25697]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:07:39.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:38 smithi086 ceph-mon[24811]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:07:39.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:38 smithi086 ceph-mon[29505]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:07:39.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:38 smithi192 ceph-mon[25697]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:07:40.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:39 smithi086 ceph-mon[24811]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 896 B/s rd, 0 op/s 2024-04-02T05:07:40.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:39 smithi086 ceph-mon[29505]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 896 B/s rd, 0 op/s 2024-04-02T05:07:40.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:39 smithi192 ceph-mon[25697]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 896 B/s rd, 0 op/s 2024-04-02T05:07:42.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:41 smithi086 ceph-mon[29505]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 680 B/s rd, 0 op/s 2024-04-02T05:07:42.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:41 smithi086 ceph-mon[24811]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 680 B/s rd, 0 op/s 2024-04-02T05:07:42.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:41 smithi192 ceph-mon[25697]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 680 B/s rd, 0 op/s 2024-04-02T05:07:43.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:42 smithi086 ceph-mon[29505]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:07:43.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:42 smithi086 ceph-mon[24811]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:07:43.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:42 smithi192 ceph-mon[25697]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:07:46.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:45 smithi086 ceph-mon[29505]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T05:07:46.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:45 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:07:46.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:45 smithi086 ceph-mon[24811]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T05:07:46.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:45 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:07:46.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:45 smithi192 ceph-mon[25697]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T05:07:46.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:45 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:07:47.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:07:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:07:47] "GET /metrics HTTP/1.1" 200 37982 "" "Prometheus/2.33.4" 2024-04-02T05:07:48.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:47 smithi192 ceph-mon[25697]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:07:48.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:47 smithi086 ceph-mon[24811]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:07:48.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:47 smithi086 ceph-mon[29505]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:07:49.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:48 smithi192 ceph-mon[25697]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:07:49.234 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:48 smithi086 ceph-mon[29505]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:07:49.235 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:48 smithi086 ceph-mon[24811]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:07:50.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:49 smithi192 ceph-mon[25697]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:07:50.250 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:49 smithi086 ceph-mon[29505]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:07:50.250 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:49 smithi086 ceph-mon[24811]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:07:52.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:51 smithi192 ceph-mon[25697]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:07:52.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:51 smithi086 ceph-mon[24811]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:07:52.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:51 smithi086 ceph-mon[29505]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:07:54.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:53 smithi192 ceph-mon[25697]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:07:54.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:53 smithi086 ceph-mon[24811]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:07:54.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:53 smithi086 ceph-mon[29505]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:07:56.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:55 smithi192 ceph-mon[25697]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:07:56.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:55 smithi086 ceph-mon[24811]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:07:56.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:55 smithi086 ceph-mon[29505]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:07:57.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:07:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:07:57] "GET /metrics HTTP/1.1" 200 37982 "" "Prometheus/2.33.4" 2024-04-02T05:07:58.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:57 smithi192 ceph-mon[25697]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:07:58.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:57 smithi086 ceph-mon[24811]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:07:58.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:57 smithi086 ceph-mon[29505]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:07:59.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:58 smithi192 ceph-mon[25697]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:07:59.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:58 smithi086 ceph-mon[24811]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:07:59.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:58 smithi086 ceph-mon[29505]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:08:00.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:07:59 smithi192 ceph-mon[25697]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:00.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:07:59 smithi086 ceph-mon[24811]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:00.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:07:59 smithi086 ceph-mon[29505]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:01.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:00 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:08:01.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:00 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:08:01.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:00 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:08:02.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:01 smithi192 ceph-mon[25697]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:02.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:01 smithi086 ceph-mon[24811]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:02.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:01 smithi086 ceph-mon[29505]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:04.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:03 smithi192 ceph-mon[25697]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:08:04.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:03 smithi086 ceph-mon[24811]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:08:04.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:03 smithi086 ceph-mon[29505]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:08:06.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:05 smithi192 ceph-mon[25697]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:06.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:05 smithi086 ceph-mon[24811]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:06.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:05 smithi086 ceph-mon[29505]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:07.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:08:07 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:08:07] "GET /metrics HTTP/1.1" 200 37984 "" "Prometheus/2.33.4" 2024-04-02T05:08:08.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:07 smithi086 ceph-mon[24811]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:08:08.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:07 smithi086 ceph-mon[29505]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:08:08.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:07 smithi192 ceph-mon[25697]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:08:09.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:08 smithi086 ceph-mon[24811]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:08:09.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:08 smithi086 ceph-mon[29505]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:08:09.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:08 smithi192 ceph-mon[25697]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:08:10.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:09 smithi086 ceph-mon[24811]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:10.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:09 smithi086 ceph-mon[29505]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:10.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:09 smithi192 ceph-mon[25697]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:12.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:11 smithi086 ceph-mon[24811]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:12.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:11 smithi086 ceph-mon[29505]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:12.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:11 smithi192 ceph-mon[25697]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:13.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:12 smithi086 ceph-mon[24811]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:08:13.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:12 smithi086 ceph-mon[29505]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:08:13.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:12 smithi192 ceph-mon[25697]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:08:16.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:15 smithi086 ceph-mon[24811]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:16.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:15 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:08:16.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:15 smithi086 ceph-mon[29505]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:16.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:15 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:08:16.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:15 smithi192 ceph-mon[25697]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:16.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:15 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:08:17.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:08:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:08:17] "GET /metrics HTTP/1.1" 200 37985 "" "Prometheus/2.33.4" 2024-04-02T05:08:18.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:17 smithi192 ceph-mon[25697]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:08:18.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:17 smithi086 ceph-mon[24811]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:08:18.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:17 smithi086 ceph-mon[29505]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:08:19.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:18 smithi192 ceph-mon[25697]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:08:19.234 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:18 smithi086 ceph-mon[29505]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:08:19.235 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:18 smithi086 ceph-mon[24811]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:08:20.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:19 smithi192 ceph-mon[25697]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:20.249 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:19 smithi086 ceph-mon[24811]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:20.250 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:19 smithi086 ceph-mon[29505]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:22.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:21 smithi192 ceph-mon[25697]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:22.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:21 smithi086 ceph-mon[24811]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:22.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:21 smithi086 ceph-mon[29505]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:24.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:23 smithi192 ceph-mon[25697]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:08:24.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:23 smithi086 ceph-mon[24811]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:08:24.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:23 smithi086 ceph-mon[29505]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:08:26.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:25 smithi192 ceph-mon[25697]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:26.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:25 smithi086 ceph-mon[24811]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:26.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:25 smithi086 ceph-mon[29505]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:27.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:08:27 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:08:27] "GET /metrics HTTP/1.1" 200 37985 "" "Prometheus/2.33.4" 2024-04-02T05:08:28.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:27 smithi192 ceph-mon[25697]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:08:28.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:27 smithi086 ceph-mon[24811]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:08:28.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:27 smithi086 ceph-mon[29505]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:08:29.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:28 smithi192 ceph-mon[25697]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:08:29.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:28 smithi086 ceph-mon[24811]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:08:29.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:28 smithi086 ceph-mon[29505]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:08:30.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:29 smithi192 ceph-mon[25697]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:30.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:29 smithi086 ceph-mon[29505]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:30.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:29 smithi086 ceph-mon[24811]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:31.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:30 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:08:31.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:30 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:08:31.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:30 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:08:32.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:31 smithi086 ceph-mon[29505]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:32.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:31 smithi086 ceph-mon[24811]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:32.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:31 smithi192 ceph-mon[25697]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:34.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:33 smithi086 ceph-mon[29505]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:08:34.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:33 smithi086 ceph-mon[24811]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:08:34.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:33 smithi192 ceph-mon[25697]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:08:36.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:35 smithi086 ceph-mon[29505]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:36.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:35 smithi086 ceph-mon[24811]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:36.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:35 smithi192 ceph-mon[25697]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:37.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:08:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:08:37] "GET /metrics HTTP/1.1" 200 37983 "" "Prometheus/2.33.4" 2024-04-02T05:08:38.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:37 smithi086 ceph-mon[24811]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:08:38.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:37 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:08:38.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:37 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:08:38.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:37 smithi086 ceph-mon[24811]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:08:38.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:37 smithi086 ceph-mon[29505]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:08:38.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:37 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:08:38.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:37 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:08:38.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:37 smithi086 ceph-mon[29505]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:08:38.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:37 smithi192 ceph-mon[25697]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:08:38.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:37 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:08:38.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:37 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:08:38.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:37 smithi192 ceph-mon[25697]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:08:39.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:38 smithi086 ceph-mon[29505]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 952 B/s rd, 0 op/s 2024-04-02T05:08:39.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:38 smithi086 ceph-mon[29505]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:08:39.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:38 smithi086 ceph-mon[29505]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:08:39.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:38 smithi086 ceph-mon[24811]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 952 B/s rd, 0 op/s 2024-04-02T05:08:39.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:38 smithi086 ceph-mon[24811]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:08:39.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:38 smithi086 ceph-mon[24811]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:08:39.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:38 smithi192 ceph-mon[25697]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 952 B/s rd, 0 op/s 2024-04-02T05:08:39.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:38 smithi192 ceph-mon[25697]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:08:39.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:38 smithi192 ceph-mon[25697]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:08:41.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:40 smithi086 ceph-mon[24811]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:08:41.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:40 smithi086 ceph-mon[29505]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:08:41.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:40 smithi192 ceph-mon[25697]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:08:42.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:42 smithi086 ceph-mon[29505]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:08:42.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:41 smithi086 ceph-mon[24811]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:08:42.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:42 smithi192 ceph-mon[25697]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:08:44.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:44 smithi086 ceph-mon[24811]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:08:44.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:44 smithi086 ceph-mon[29505]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:08:44.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:44 smithi192 ceph-mon[25697]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:08:45.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:45 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:08:45.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:45 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:08:45.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:45 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:08:47.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:46 smithi192 ceph-mon[25697]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 585 B/s rd, 0 op/s 2024-04-02T05:08:47.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:46 smithi086 ceph-mon[24811]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 585 B/s rd, 0 op/s 2024-04-02T05:08:47.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:46 smithi086 ceph-mon[29505]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 585 B/s rd, 0 op/s 2024-04-02T05:08:47.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:08:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:08:47] "GET /metrics HTTP/1.1" 200 37982 "" "Prometheus/2.33.4" 2024-04-02T05:08:49.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:48 smithi192 ceph-mon[25697]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:08:49.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:48 smithi192 ceph-mon[25697]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:08:49.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:48 smithi086 ceph-mon[24811]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:08:49.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:48 smithi086 ceph-mon[24811]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:08:49.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:48 smithi086 ceph-mon[29505]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:08:49.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:48 smithi086 ceph-mon[29505]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:08:51.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:50 smithi192 ceph-mon[25697]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:51.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:50 smithi086 ceph-mon[29505]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:51.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:50 smithi086 ceph-mon[24811]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:53.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:52 smithi192 ceph-mon[25697]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:08:53.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:52 smithi086 ceph-mon[29505]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:08:53.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:52 smithi086 ceph-mon[24811]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:08:55.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:54 smithi192 ceph-mon[25697]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:55.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:54 smithi086 ceph-mon[29505]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:55.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:54 smithi086 ceph-mon[24811]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:57.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:56 smithi192 ceph-mon[25697]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:57.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:56 smithi086 ceph-mon[29505]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:57.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:56 smithi086 ceph-mon[24811]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:08:57.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:08:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:08:57] "GET /metrics HTTP/1.1" 200 37982 "" "Prometheus/2.33.4" 2024-04-02T05:08:59.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:58 smithi192 ceph-mon[25697]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:08:59.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:08:58 smithi192 ceph-mon[25697]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:08:59.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:58 smithi086 ceph-mon[24811]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:08:59.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:08:58 smithi086 ceph-mon[24811]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:08:59.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:58 smithi086 ceph-mon[29505]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:08:59.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:08:58 smithi086 ceph-mon[29505]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:09:01.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:00 smithi192 ceph-mon[25697]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:01.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:00 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:09:01.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:00 smithi086 ceph-mon[24811]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:01.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:00 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:09:01.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:00 smithi086 ceph-mon[29505]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:01.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:00 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:09:03.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:02 smithi192 ceph-mon[25697]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:03.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:02 smithi086 ceph-mon[24811]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:03.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:02 smithi086 ceph-mon[29505]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:05.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:04 smithi192 ceph-mon[25697]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:05.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:04 smithi086 ceph-mon[24811]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:05.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:04 smithi086 ceph-mon[29505]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:07.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:06 smithi192 ceph-mon[25697]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:07.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:06 smithi086 ceph-mon[24811]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:07.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:06 smithi086 ceph-mon[29505]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:07.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:09:07 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:09:07] "GET /metrics HTTP/1.1" 200 37983 "" "Prometheus/2.33.4" 2024-04-02T05:09:09.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:08 smithi192 ceph-mon[25697]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:09.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:08 smithi192 ceph-mon[25697]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:09:09.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:08 smithi086 ceph-mon[24811]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:09.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:08 smithi086 ceph-mon[24811]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:09:09.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:08 smithi086 ceph-mon[29505]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:09.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:08 smithi086 ceph-mon[29505]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:09:11.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:10 smithi086 ceph-mon[24811]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:11.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:10 smithi086 ceph-mon[29505]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:11.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:10 smithi192 ceph-mon[25697]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:13.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:12 smithi086 ceph-mon[24811]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:13.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:12 smithi086 ceph-mon[29505]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:13.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:12 smithi192 ceph-mon[25697]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:15.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:14 smithi086 ceph-mon[24811]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:15.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:14 smithi086 ceph-mon[29505]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:15.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:14 smithi192 ceph-mon[25697]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:16.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:15 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:09:16.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:15 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:09:16.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:15 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:09:17.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:16 smithi086 ceph-mon[29505]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:17.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:16 smithi086 ceph-mon[24811]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:17.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:16 smithi192 ceph-mon[25697]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:17.427 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:09:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:09:17] "GET /metrics HTTP/1.1" 200 37981 "" "Prometheus/2.33.4" 2024-04-02T05:09:19.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:18 smithi086 ceph-mon[24811]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:19.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:18 smithi086 ceph-mon[24811]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:09:19.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:18 smithi086 ceph-mon[29505]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:19.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:18 smithi086 ceph-mon[29505]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:09:19.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:18 smithi192 ceph-mon[25697]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:19.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:18 smithi192 ceph-mon[25697]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:09:21.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:20 smithi086 ceph-mon[24811]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:21.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:20 smithi086 ceph-mon[29505]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:21.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:20 smithi192 ceph-mon[25697]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:22.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:22 smithi086 ceph-mon[24811]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:22.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:22 smithi086 ceph-mon[29505]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:22.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:22 smithi192 ceph-mon[25697]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:24.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:24 smithi086 ceph-mon[29505]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:24.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:24 smithi086 ceph-mon[24811]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:24.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:24 smithi192 ceph-mon[25697]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:27.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:26 smithi192 ceph-mon[25697]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:27.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:26 smithi086 ceph-mon[29505]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:27.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:26 smithi086 ceph-mon[24811]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:27.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:09:27 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:09:27] "GET /metrics HTTP/1.1" 200 37981 "" "Prometheus/2.33.4" 2024-04-02T05:09:29.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:28 smithi192 ceph-mon[25697]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:29.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:28 smithi192 ceph-mon[25697]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:09:29.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:28 smithi086 ceph-mon[29505]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:29.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:28 smithi086 ceph-mon[29505]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:09:29.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:28 smithi086 ceph-mon[24811]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:29.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:28 smithi086 ceph-mon[24811]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:09:31.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:30 smithi192 ceph-mon[25697]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:31.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:30 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:09:31.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:30 smithi086 ceph-mon[24811]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:31.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:30 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:09:31.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:30 smithi086 ceph-mon[29505]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:31.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:30 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:09:33.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:32 smithi192 ceph-mon[25697]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:33.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:32 smithi086 ceph-mon[29505]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:33.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:32 smithi086 ceph-mon[24811]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:35.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:34 smithi192 ceph-mon[25697]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:35.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:34 smithi086 ceph-mon[29505]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:35.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:34 smithi086 ceph-mon[24811]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:37.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:36 smithi192 ceph-mon[25697]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:37.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:36 smithi086 ceph-mon[29505]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:37.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:36 smithi086 ceph-mon[24811]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:37.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:09:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:09:37] "GET /metrics HTTP/1.1" 200 37994 "" "Prometheus/2.33.4" 2024-04-02T05:09:39.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:38 smithi192 ceph-mon[25697]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:39.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:38 smithi192 ceph-mon[25697]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:09:39.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:38 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:09:39.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:38 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:09:39.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:38 smithi192 ceph-mon[25697]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:09:39.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:38 smithi086 ceph-mon[24811]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:39.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:38 smithi086 ceph-mon[24811]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:09:39.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:38 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:09:39.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:38 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:09:39.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:38 smithi086 ceph-mon[24811]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:09:39.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:38 smithi086 ceph-mon[29505]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:39.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:38 smithi086 ceph-mon[29505]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:09:39.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:38 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:09:39.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:38 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:09:39.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:38 smithi086 ceph-mon[29505]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:09:40.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:39 smithi086 ceph-mon[24811]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 958 B/s rd, 0 op/s 2024-04-02T05:09:40.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:39 smithi086 ceph-mon[24811]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:40.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:39 smithi086 ceph-mon[29505]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 958 B/s rd, 0 op/s 2024-04-02T05:09:40.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:39 smithi086 ceph-mon[29505]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:40.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:39 smithi192 ceph-mon[25697]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 958 B/s rd, 0 op/s 2024-04-02T05:09:40.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:39 smithi192 ceph-mon[25697]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:42.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:41 smithi086 ceph-mon[24811]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 589 B/s rd, 0 op/s 2024-04-02T05:09:42.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:41 smithi086 ceph-mon[29505]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 589 B/s rd, 0 op/s 2024-04-02T05:09:42.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:41 smithi192 ceph-mon[25697]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 589 B/s rd, 0 op/s 2024-04-02T05:09:44.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:43 smithi086 ceph-mon[24811]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:44.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:43 smithi086 ceph-mon[29505]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:44.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:43 smithi192 ceph-mon[25697]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:46.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:45 smithi086 ceph-mon[24811]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:46.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:45 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:09:46.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:45 smithi086 ceph-mon[29505]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:46.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:45 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:09:46.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:45 smithi192 ceph-mon[25697]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:46.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:45 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:09:47.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:09:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:09:47] "GET /metrics HTTP/1.1" 200 37988 "" "Prometheus/2.33.4" 2024-04-02T05:09:48.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:47 smithi086 ceph-mon[24811]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:48.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:47 smithi086 ceph-mon[29505]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:48.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:47 smithi192 ceph-mon[25697]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:49.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:48 smithi086 ceph-mon[24811]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:09:49.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:48 smithi086 ceph-mon[29505]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:09:49.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:48 smithi192 ceph-mon[25697]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:09:50.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:49 smithi086 ceph-mon[24811]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:09:50.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:49 smithi086 ceph-mon[29505]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:09:50.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:49 smithi192 ceph-mon[25697]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:09:52.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:51 smithi086 ceph-mon[29505]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:52.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:51 smithi086 ceph-mon[24811]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:52.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:51 smithi192 ceph-mon[25697]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:53.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:52 smithi086 ceph-mon[29505]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:53.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:52 smithi086 ceph-mon[24811]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:53.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:52 smithi192 ceph-mon[25697]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:55.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:55 smithi192 ceph-mon[25697]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:55.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:55 smithi086 ceph-mon[29505]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:55.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:55 smithi086 ceph-mon[24811]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:09:57.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:09:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:09:57] "GET /metrics HTTP/1.1" 200 37988 "" "Prometheus/2.33.4" 2024-04-02T05:09:58.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:57 smithi086 ceph-mon[29505]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:58.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:57 smithi086 ceph-mon[24811]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:58.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:57 smithi192 ceph-mon[25697]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:09:59.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:58 smithi192 ceph-mon[25697]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:09:59.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:58 smithi086 ceph-mon[29505]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:09:59.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:58 smithi086 ceph-mon[24811]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:10:00.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:09:59 smithi192 ceph-mon[25697]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:10:00.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:09:59 smithi086 ceph-mon[29505]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:10:00.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:09:59 smithi086 ceph-mon[24811]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:10:01.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:00 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:10:01.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:00 smithi192 ceph-mon[25697]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-02T05:10:01.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:00 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:10:01.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:00 smithi086 ceph-mon[29505]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-02T05:10:01.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:00 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:10:01.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:00 smithi086 ceph-mon[24811]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-02T05:10:02.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:01 smithi192 ceph-mon[25697]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:10:02.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:01 smithi086 ceph-mon[24811]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:10:02.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:01 smithi086 ceph-mon[29505]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:10:04.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:03 smithi192 ceph-mon[25697]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:10:04.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:03 smithi086 ceph-mon[24811]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:10:04.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:03 smithi086 ceph-mon[29505]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:10:06.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:05 smithi086 ceph-mon[29505]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:10:06.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:05 smithi086 ceph-mon[24811]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:10:06.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:05 smithi192 ceph-mon[25697]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:10:07.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:07 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:10:07] "GET /metrics HTTP/1.1" 200 37992 "" "Prometheus/2.33.4" 2024-04-02T05:10:08.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:07 smithi086 ceph-mon[24811]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:10:08.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:07 smithi086 ceph-mon[29505]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:10:08.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:07 smithi192 ceph-mon[25697]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:10:09.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:08 smithi086 ceph-mon[24811]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:10:09.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:08 smithi086 ceph-mon[29505]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:10:09.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:08 smithi192 ceph-mon[25697]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:10:10.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:09 smithi086 ceph-mon[29505]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:10:10.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:09 smithi086 ceph-mon[24811]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:10:10.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:09 smithi192 ceph-mon[25697]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:10:12.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:11 smithi086 ceph-mon[29505]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:10:12.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:11 smithi086 ceph-mon[24811]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:10:12.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:11 smithi192 ceph-mon[25697]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:10:14.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:13 smithi086 ceph-mon[29505]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:10:14.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:13 smithi086 ceph-mon[24811]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:10:14.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:13 smithi192 ceph-mon[25697]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:10:16.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:15 smithi086 ceph-mon[29505]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:10:16.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:15 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:10:16.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:15 smithi086 ceph-mon[24811]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:10:16.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:15 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:10:16.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:15 smithi192 ceph-mon[25697]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:10:16.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:15 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:10:17.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:17 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:10:17] "GET /metrics HTTP/1.1" 200 37990 "" "Prometheus/2.33.4" 2024-04-02T05:10:18.325 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:17 smithi086 ceph-mon[29505]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:10:18.325 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:17 smithi086 ceph-mon[24811]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:10:18.355 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch ps' 2024-04-02T05:10:18.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:17 smithi192 ceph-mon[25697]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:10:19.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:18 smithi086 ceph-mon[24811]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:10:19.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:18 smithi086 ceph-mon[29505]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:10:19.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:18 smithi192 ceph-mon[25697]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:10:20.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:19 smithi086 ceph-mon[24811]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:10:20.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:19 smithi086 ceph-mon[29505]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:10:20.366 INFO:teuthology.orchestra.run.smithi086.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T05:10:20.366 INFO:teuthology.orchestra.run.smithi086.stdout:alertmanager.a smithi086 *:9093,9094 running (12m) 2m ago 20m 22.5M - 0.23.0 ba2b418f427c b2769c8e287a 2024-04-02T05:10:20.366 INFO:teuthology.orchestra.run.smithi086.stdout:grafana.a smithi192 *:3000 error 2m ago 20m - - 2024-04-02T05:10:20.366 INFO:teuthology.orchestra.run.smithi086.stdout:iscsi.foo.smithi086.rsbqwx smithi086 running (2m) 2m ago 19m 87.1M - 3.5 e1d6a67b021e 2b9643017ea8 2024-04-02T05:10:20.366 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.x smithi192 *:8443,9283 running (15m) 2m ago 24m 495M - 19.0.0-1979-g6cec793d 78f830190839 fd618149d6b4 2024-04-02T05:10:20.366 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.y smithi086 *:8443,9283,8765 running (9m) 2m ago 26m 447M - 19.0.0-1979-g6cec793d 78f830190839 3dca54a018ab 2024-04-02T05:10:20.366 INFO:teuthology.orchestra.run.smithi086.stdout:mon.a smithi086 running (26m) 2m ago 26m 70.9M 2048M 17.2.0 e1d6a67b021e b6c507ab96f8 2024-04-02T05:10:20.366 INFO:teuthology.orchestra.run.smithi086.stdout:mon.b smithi192 running (24m) 2m ago 24m 54.8M 2048M 17.2.0 e1d6a67b021e 4dadd8d1c318 2024-04-02T05:10:20.366 INFO:teuthology.orchestra.run.smithi086.stdout:mon.c smithi086 running (25m) 2m ago 25m 51.2M 2048M 17.2.0 e1d6a67b021e 49275483a2d4 2024-04-02T05:10:20.366 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.a smithi086 *:9100 running (12m) 2m ago 21m 19.5M - 1.3.1 1dbe0e931976 843a5e9887fb 2024-04-02T05:10:20.366 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.b smithi192 *:9100 running (12m) 2m ago 20m 17.2M - 1.3.1 1dbe0e931976 bd849d6c4fb0 2024-04-02T05:10:20.366 INFO:teuthology.orchestra.run.smithi086.stdout:osd.0 smithi086 running (24m) 2m ago 24m 56.5M 2992M 17.2.0 e1d6a67b021e 6d90c60d1c71 2024-04-02T05:10:20.367 INFO:teuthology.orchestra.run.smithi086.stdout:osd.1 smithi086 running (24m) 2m ago 24m 54.9M 2992M 17.2.0 e1d6a67b021e 203689f71603 2024-04-02T05:10:20.367 INFO:teuthology.orchestra.run.smithi086.stdout:osd.2 smithi086 running (23m) 2m ago 23m 49.6M 2992M 17.2.0 e1d6a67b021e 472c860a9af8 2024-04-02T05:10:20.367 INFO:teuthology.orchestra.run.smithi086.stdout:osd.3 smithi086 running (23m) 2m ago 23m 52.2M 2992M 17.2.0 e1d6a67b021e 327538dac44e 2024-04-02T05:10:20.367 INFO:teuthology.orchestra.run.smithi086.stdout:osd.4 smithi192 running (22m) 2m ago 22m 53.7M 3248M 17.2.0 e1d6a67b021e 6535d7919a2d 2024-04-02T05:10:20.367 INFO:teuthology.orchestra.run.smithi086.stdout:osd.5 smithi192 running (22m) 2m ago 22m 53.6M 3248M 17.2.0 e1d6a67b021e 7259249605f2 2024-04-02T05:10:20.367 INFO:teuthology.orchestra.run.smithi086.stdout:osd.6 smithi192 running (22m) 2m ago 22m 50.9M 3248M 17.2.0 e1d6a67b021e bb7d171b02a1 2024-04-02T05:10:20.367 INFO:teuthology.orchestra.run.smithi086.stdout:osd.7 smithi192 running (21m) 2m ago 21m 53.9M 3248M 17.2.0 e1d6a67b021e 95f5a0353999 2024-04-02T05:10:20.367 INFO:teuthology.orchestra.run.smithi086.stdout:prometheus.a smithi192 *:9095 running (2m) 2m ago 21m 46.7M - 2.33.4 514e6a882f6e f455b2541990 2024-04-02T05:10:20.367 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi086.bxzxlp smithi086 *:8000 running (19m) 2m ago 19m 76.7M - 17.2.0 e1d6a67b021e 6725785fbc53 2024-04-02T05:10:20.367 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi192.pmlbot smithi192 *:8000 running (19m) 2m ago 19m 75.9M - 17.2.0 e1d6a67b021e 5ae3638f8b1c 2024-04-02T05:10:20.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:19 smithi192 ceph-mon[25697]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:10:20.945 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch ls' 2024-04-02T05:10:22.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:21 smithi192 ceph-mon[25697]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:10:22.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:21 smithi192 ceph-mon[25697]: from='client.25318 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:10:22.259 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:21 smithi086 ceph-mon[24811]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:10:22.260 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:21 smithi086 ceph-mon[24811]: from='client.25318 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:10:22.260 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:21 smithi086 ceph-mon[29505]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:10:22.260 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:21 smithi086 ceph-mon[29505]: from='client.25318 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:10:22.998 INFO:teuthology.orchestra.run.smithi086.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-04-02T05:10:22.999 INFO:teuthology.orchestra.run.smithi086.stdout:alertmanager ?:9093,9094 1/1 2m ago 21m smithi086=a;count:1 2024-04-02T05:10:22.999 INFO:teuthology.orchestra.run.smithi086.stdout:grafana ?:3000 0/1 2m ago 21m smithi192=a;count:1 2024-04-02T05:10:22.999 INFO:teuthology.orchestra.run.smithi086.stdout:iscsi.foo ?:5000 1/1 2m ago 19m count:1 2024-04-02T05:10:22.999 INFO:teuthology.orchestra.run.smithi086.stdout:mgr 2/2 2m ago 24m smithi086=y;smithi192=x;count:2 2024-04-02T05:10:22.999 INFO:teuthology.orchestra.run.smithi086.stdout:mon 3/3 2m ago 25m smithi086:172.21.15.86=a;smithi086:[v2:172.21.15.86:3301,v1:172.21.15.86:6790]=c;smithi192:172.21.15.192=b;count:3 2024-04-02T05:10:22.999 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter ?:9100 2/2 2m ago 21m smithi086=a;smithi192=b;count:2 2024-04-02T05:10:22.999 INFO:teuthology.orchestra.run.smithi086.stdout:osd 8 2m ago - 2024-04-02T05:10:22.999 INFO:teuthology.orchestra.run.smithi086.stdout:prometheus ?:9095 1/1 2m ago 21m smithi192=a;count:1 2024-04-02T05:10:22.999 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo ?:8000 2/2 2m ago 19m count:2 2024-04-02T05:10:23.575 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph versions' 2024-04-02T05:10:24.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:23 smithi192 ceph-mon[25697]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:10:24.213 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:23 smithi086 ceph-mon[29505]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:10:24.213 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:23 smithi086 ceph-mon[24811]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:10:25.154 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:24 smithi086 ceph-mon[29505]: from='client.25324 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:10:25.154 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:24 smithi086 ceph-mon[24811]: from='client.25324 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:10:25.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:24 smithi192 ceph-mon[25697]: from='client.25324 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:10:25.652 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:10:25.652 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": { 2024-04-02T05:10:25.652 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-02T05:10:25.652 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:10:25.652 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": { 2024-04-02T05:10:25.652 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 2 2024-04-02T05:10:25.652 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:10:25.652 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": { 2024-04-02T05:10:25.653 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-02T05:10:25.653 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:10:25.653 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": {}, 2024-04-02T05:10:25.653 INFO:teuthology.orchestra.run.smithi086.stdout: "rgw": { 2024-04-02T05:10:25.653 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-02T05:10:25.653 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:10:25.653 INFO:teuthology.orchestra.run.smithi086.stdout: "overall": { 2024-04-02T05:10:25.653 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-02T05:10:25.653 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 2 2024-04-02T05:10:25.653 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-04-02T05:10:25.653 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:10:26.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:25 smithi192 ceph-mon[25697]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:10:26.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:25 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/445236149' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:10:26.273 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:25 smithi086 ceph-mon[24811]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:10:26.274 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:25 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/445236149' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:10:26.274 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:25 smithi086 ceph-mon[29505]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:10:26.274 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:25 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/445236149' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:10:26.297 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr' 2024-04-02T05:10:27.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:27 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:10:27] "GET /metrics HTTP/1.1" 200 37990 "" "Prometheus/2.33.4" 2024-04-02T05:10:28.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:27 smithi086 ceph-mon[24811]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:10:28.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:27 smithi086 ceph-mon[29505]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:10:28.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:27 smithi192 ceph-mon[25697]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:10:29.053 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:28 smithi086 ceph-mon[24811]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:10:29.053 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:28 smithi086 ceph-mon[29505]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:10:29.329 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:28 smithi192 ceph-mon[25697]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:10:29.809 INFO:teuthology.orchestra.run.smithi086.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:10:30.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:29 smithi086 ceph-mon[24811]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:10:30.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:29 smithi086 ceph-mon[24811]: from='client.25336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:10:30.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:29 smithi086 ceph-mon[24811]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:10:30.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:29 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:10:30.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:29 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:10:30.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:29 smithi086 ceph-mon[24811]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:10:30.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:29 smithi086 ceph-mon[29505]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:10:30.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:29 smithi086 ceph-mon[29505]: from='client.25336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:10:30.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:29 smithi086 ceph-mon[29505]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:10:30.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:29 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:10:30.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:29 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:10:30.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:29 smithi086 ceph-mon[29505]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:10:30.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:29 smithi192 ceph-mon[25697]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:10:30.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:29 smithi192 ceph-mon[25697]: from='client.25336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:10:30.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:29 smithi192 ceph-mon[25697]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:10:30.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:29 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:10:30.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:29 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:10:30.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:29 smithi192 ceph-mon[25697]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:10:30.586 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-02T05:10:31.308 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:30 smithi086 ceph-mon[29505]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:10:31.308 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:30 smithi086 ceph-mon[29505]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 871 B/s rd, 0 op/s 2024-04-02T05:10:31.308 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:30 smithi086 ceph-mon[29505]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T05:10:31.308 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:30 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:10:31.308 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:30 smithi086 ceph-mon[24811]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:10:31.308 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:30 smithi086 ceph-mon[24811]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 871 B/s rd, 0 op/s 2024-04-02T05:10:31.308 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:30 smithi086 ceph-mon[24811]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T05:10:31.308 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:30 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:10:31.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:30 smithi192 ceph-mon[25697]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:10:31.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:30 smithi192 ceph-mon[25697]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 871 B/s rd, 0 op/s 2024-04-02T05:10:31.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:30 smithi192 ceph-mon[25697]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T05:10:31.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:30 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:10:32.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:31 smithi086 ceph-mon[24811]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:10:32.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:31 smithi086 ceph-mon[24811]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:10:32.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:31 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:10:32.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:31 smithi086 ceph-mon[24811]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-02T05:10:32.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:31 smithi086 ceph-mon[24811]: from='mgr.15255 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-02T05:10:32.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:31 smithi086 ceph-mon[24811]: osdmap e110: 8 total, 8 up, 8 in 2024-04-02T05:10:32.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:31 smithi086 ceph-mon[29505]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:10:32.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:31 smithi086 ceph-mon[29505]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:10:32.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:31 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:10:32.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:31 smithi086 ceph-mon[29505]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-02T05:10:32.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:31 smithi086 ceph-mon[29505]: from='mgr.15255 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-02T05:10:32.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:31 smithi086 ceph-mon[29505]: osdmap e110: 8 total, 8 up, 8 in 2024-04-02T05:10:32.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:31 smithi192 ceph-mon[25697]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:10:32.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:31 smithi192 ceph-mon[25697]: from='mgr.15255 ' entity='mgr.x' 2024-04-02T05:10:32.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:31 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:10:32.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:31 smithi192 ceph-mon[25697]: from='mgr.15255 172.21.15.192:0/3215166203' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-02T05:10:32.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:31 smithi192 ceph-mon[25697]: from='mgr.15255 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-02T05:10:32.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:31 smithi192 ceph-mon[25697]: osdmap e110: 8 total, 8 up, 8 in 2024-04-02T05:10:32.717 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: [02/Apr/2024:05:10:32] ENGINE Bus STOPPING 2024-04-02T05:10:32.717 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: [02/Apr/2024:05:10:32] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-02T05:10:32.717 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: [02/Apr/2024:05:10:32] ENGINE Bus STOPPED 2024-04-02T05:10:32.945 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:10:32.685+0000 7f7115d30700 -1 mgr handle_mgr_map I was active but no longer am 2024-04-02T05:10:32.945 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ignoring --setuser ceph since I am not root 2024-04-02T05:10:32.945 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: ignoring --setgroup ceph since I am not root 2024-04-02T05:10:32.946 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-mgr[45429]: -- 172.21.15.192:0/749541741 <== mon.2 v2:172.21.15.192:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55ca1e75a5a0 con 0x55ca1e72e800 2024-04-02T05:10:32.946 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:10:32.712Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.192:8765/sd/prometheus/sd-config?service=node-exporter\": read tcp 172.21.15.192:38082->172.21.15.192:8765: read: connection reset by peer" 2024-04-02T05:10:32.946 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:10:32.712Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.192:8765/sd/prometheus/sd-config?service=ceph-exporter\": read tcp 172.21.15.192:38110->172.21.15.192:8765: read: connection reset by peer" 2024-04-02T05:10:32.946 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:10:32.712Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.192:8765/sd/prometheus/sd-config?service=mgr-prometheus\": read tcp 172.21.15.192:38120->172.21.15.192:8765: read: connection reset by peer" 2024-04-02T05:10:32.946 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:10:32.712Z caller=refresh.go:98 level=error component="discovery manager notify" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.192:8765/sd/prometheus/sd-config?service=alertmanager\": read tcp 172.21.15.192:38096->172.21.15.192:8765: read: connection reset by peer" 2024-04-02T05:10:33.035 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[29505]: Upgrade: Target is version 19.0.0-1979-g6cec793d (squid) 2024-04-02T05:10:33.035 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[29505]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7bd5b84926d4a0d6c51e5562e965bff211cec2740c9f4dec9b5e00a2bbb75eae'] 2024-04-02T05:10:33.035 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[29505]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-02T05:10:33.035 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[29505]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-02T05:10:33.035 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[29505]: Failing over to other MGR 2024-04-02T05:10:33.035 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[29505]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-02T05:10:33.035 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[29505]: Standby manager daemon x started 2024-04-02T05:10:33.035 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[29505]: from='mgr.15255 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-02T05:10:33.035 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[29505]: mgrmap e36: y(active, starting, since 1.00163s), standbys: x 2024-04-02T05:10:33.035 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T05:10:33.036 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T05:10:33.036 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T05:10:33.036 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T05:10:33.036 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T05:10:33.036 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T05:10:33.036 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T05:10:33.036 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T05:10:33.036 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T05:10:33.036 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T05:10:33.036 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T05:10:33.036 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T05:10:33.036 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T05:10:33.036 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T05:10:33.036 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T05:10:33.036 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T05:10:33.037 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[29505]: Manager daemon y is now available 2024-04-02T05:10:33.037 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:10:33.037 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T05:10:33.037 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T05:10:33.037 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T05:10:33.037 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T05:10:33.037 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: [02/Apr/2024:05:10:32] ENGINE Bus STARTING 2024-04-02T05:10:33.037 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: [02/Apr/2024:05:10:32] ENGINE Serving on http://:::9283 2024-04-02T05:10:33.037 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: [02/Apr/2024:05:10:32] ENGINE Bus STARTED 2024-04-02T05:10:33.038 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[24811]: Upgrade: Target is version 19.0.0-1979-g6cec793d (squid) 2024-04-02T05:10:33.038 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[24811]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7bd5b84926d4a0d6c51e5562e965bff211cec2740c9f4dec9b5e00a2bbb75eae'] 2024-04-02T05:10:33.038 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[24811]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-02T05:10:33.038 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[24811]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-02T05:10:33.038 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[24811]: Failing over to other MGR 2024-04-02T05:10:33.038 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[24811]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-02T05:10:33.038 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[24811]: Standby manager daemon x started 2024-04-02T05:10:33.038 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[24811]: from='mgr.15255 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-02T05:10:33.038 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[24811]: mgrmap e36: y(active, starting, since 1.00163s), standbys: x 2024-04-02T05:10:33.038 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T05:10:33.039 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T05:10:33.039 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T05:10:33.039 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T05:10:33.039 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T05:10:33.039 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T05:10:33.039 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T05:10:33.039 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T05:10:33.039 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T05:10:33.039 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T05:10:33.039 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T05:10:33.039 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T05:10:33.040 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T05:10:33.040 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T05:10:33.040 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T05:10:33.040 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T05:10:33.040 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[24811]: Manager daemon y is now available 2024-04-02T05:10:33.040 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:10:33.040 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T05:10:33.040 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T05:10:33.040 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T05:10:33.040 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:32 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T05:10:33.264 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-mon[25697]: Upgrade: Target is version 19.0.0-1979-g6cec793d (squid) 2024-04-02T05:10:33.265 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-mon[25697]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7bd5b84926d4a0d6c51e5562e965bff211cec2740c9f4dec9b5e00a2bbb75eae'] 2024-04-02T05:10:33.265 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-mon[25697]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-02T05:10:33.265 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-mon[25697]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-02T05:10:33.265 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-mon[25697]: Failing over to other MGR 2024-04-02T05:10:33.265 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-mon[25697]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-02T05:10:33.265 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-mon[25697]: Standby manager daemon x started 2024-04-02T05:10:33.265 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-mon[25697]: from='mgr.15255 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-02T05:10:33.265 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-mon[25697]: mgrmap e36: y(active, starting, since 1.00163s), standbys: x 2024-04-02T05:10:33.266 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T05:10:33.266 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T05:10:33.266 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T05:10:33.266 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T05:10:33.266 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T05:10:33.266 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T05:10:33.267 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T05:10:33.267 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T05:10:33.267 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T05:10:33.267 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T05:10:33.267 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T05:10:33.267 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T05:10:33.267 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T05:10:33.267 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T05:10:33.267 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T05:10:33.268 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T05:10:33.268 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-mon[25697]: Manager daemon y is now available 2024-04-02T05:10:33.268 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:10:33.268 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T05:10:33.268 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T05:10:33.268 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T05:10:33.268 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:32 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T05:10:33.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:33 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:10:33.261+0000 7f1bf509b200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-02T05:10:33.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:33 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:10:33.357+0000 7f1bf509b200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-02T05:10:33.725 INFO:teuthology.orchestra.run.smithi086.stdout:true 2024-04-02T05:10:34.177 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:34 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:10:34.034+0000 7f1bf509b200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-02T05:10:34.177 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:34 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:10:34.127+0000 7f1bf509b200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-02T05:10:34.333 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:10:34 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[59010]: level=warn ts=2024-04-02T05:10:34.007Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi192.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.192:8443: connect: connection refused" 2024-04-02T05:10:34.415 INFO:teuthology.orchestra.run.smithi086.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T05:10:34.415 INFO:teuthology.orchestra.run.smithi086.stdout:alertmanager.a smithi086 *:9093,9094 running (12m) 2m ago 20m 22.5M - 0.23.0 ba2b418f427c b2769c8e287a 2024-04-02T05:10:34.415 INFO:teuthology.orchestra.run.smithi086.stdout:grafana.a smithi192 *:3000 error 2m ago 20m - - 2024-04-02T05:10:34.415 INFO:teuthology.orchestra.run.smithi086.stdout:iscsi.foo.smithi086.rsbqwx smithi086 running (3m) 2m ago 19m 87.1M - 3.5 e1d6a67b021e 2b9643017ea8 2024-04-02T05:10:34.415 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.x smithi192 *:8443,9283 running (15m) 2m ago 24m 495M - 19.0.0-1979-g6cec793d 78f830190839 fd618149d6b4 2024-04-02T05:10:34.415 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.y smithi086 *:8443,9283,8765 running (9m) 2m ago 26m 447M - 19.0.0-1979-g6cec793d 78f830190839 3dca54a018ab 2024-04-02T05:10:34.415 INFO:teuthology.orchestra.run.smithi086.stdout:mon.a smithi086 running (26m) 2m ago 26m 70.9M 2048M 17.2.0 e1d6a67b021e b6c507ab96f8 2024-04-02T05:10:34.415 INFO:teuthology.orchestra.run.smithi086.stdout:mon.b smithi192 running (25m) 2m ago 25m 54.8M 2048M 17.2.0 e1d6a67b021e 4dadd8d1c318 2024-04-02T05:10:34.415 INFO:teuthology.orchestra.run.smithi086.stdout:mon.c smithi086 running (25m) 2m ago 25m 51.2M 2048M 17.2.0 e1d6a67b021e 49275483a2d4 2024-04-02T05:10:34.415 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.a smithi086 *:9100 running (12m) 2m ago 21m 19.5M - 1.3.1 1dbe0e931976 843a5e9887fb 2024-04-02T05:10:34.415 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.b smithi192 *:9100 running (12m) 2m ago 21m 17.2M - 1.3.1 1dbe0e931976 bd849d6c4fb0 2024-04-02T05:10:34.415 INFO:teuthology.orchestra.run.smithi086.stdout:osd.0 smithi086 running (24m) 2m ago 24m 56.5M 2992M 17.2.0 e1d6a67b021e 6d90c60d1c71 2024-04-02T05:10:34.416 INFO:teuthology.orchestra.run.smithi086.stdout:osd.1 smithi086 running (24m) 2m ago 24m 54.9M 2992M 17.2.0 e1d6a67b021e 203689f71603 2024-04-02T05:10:34.416 INFO:teuthology.orchestra.run.smithi086.stdout:osd.2 smithi086 running (23m) 2m ago 23m 49.6M 2992M 17.2.0 e1d6a67b021e 472c860a9af8 2024-04-02T05:10:34.416 INFO:teuthology.orchestra.run.smithi086.stdout:osd.3 smithi086 running (23m) 2m ago 23m 52.2M 2992M 17.2.0 e1d6a67b021e 327538dac44e 2024-04-02T05:10:34.416 INFO:teuthology.orchestra.run.smithi086.stdout:osd.4 smithi192 running (23m) 2m ago 23m 53.7M 3248M 17.2.0 e1d6a67b021e 6535d7919a2d 2024-04-02T05:10:34.416 INFO:teuthology.orchestra.run.smithi086.stdout:osd.5 smithi192 running (22m) 2m ago 22m 53.6M 3248M 17.2.0 e1d6a67b021e 7259249605f2 2024-04-02T05:10:34.416 INFO:teuthology.orchestra.run.smithi086.stdout:osd.6 smithi192 running (22m) 2m ago 22m 50.9M 3248M 17.2.0 e1d6a67b021e bb7d171b02a1 2024-04-02T05:10:34.416 INFO:teuthology.orchestra.run.smithi086.stdout:osd.7 smithi192 running (21m) 2m ago 21m 53.9M 3248M 17.2.0 e1d6a67b021e 95f5a0353999 2024-04-02T05:10:34.416 INFO:teuthology.orchestra.run.smithi086.stdout:prometheus.a smithi192 *:9095 running (3m) 2m ago 21m 46.7M - 2.33.4 514e6a882f6e f455b2541990 2024-04-02T05:10:34.416 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi086.bxzxlp smithi086 *:8000 running (19m) 2m ago 19m 76.7M - 17.2.0 e1d6a67b021e 6725785fbc53 2024-04-02T05:10:34.416 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi192.pmlbot smithi192 *:8000 running (19m) 2m ago 19m 75.9M - 17.2.0 e1d6a67b021e 5ae3638f8b1c 2024-04-02T05:10:34.657 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:34 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:10:34.277+0000 7f1bf509b200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-02T05:10:34.657 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:34 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:10:34.384+0000 7f1bf509b200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-02T05:10:34.657 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:34 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:10:34.468+0000 7f1bf509b200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-02T05:10:34.828 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:10:34.828 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": { 2024-04-02T05:10:34.828 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-02T05:10:34.828 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:10:34.828 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": { 2024-04-02T05:10:34.828 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 2 2024-04-02T05:10:34.828 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:10:34.828 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": { 2024-04-02T05:10:34.828 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-02T05:10:34.829 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:10:34.829 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": {}, 2024-04-02T05:10:34.829 INFO:teuthology.orchestra.run.smithi086.stdout: "rgw": { 2024-04-02T05:10:34.829 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-02T05:10:34.829 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:10:34.829 INFO:teuthology.orchestra.run.smithi086.stdout: "overall": { 2024-04-02T05:10:34.829 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-02T05:10:34.829 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 2 2024-04-02T05:10:34.829 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-04-02T05:10:34.829 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:10:34.927 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:34 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:10:34.751+0000 7f1bf509b200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-02T05:10:34.927 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:34 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:10:34.867+0000 7f1bf509b200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-02T05:10:34.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:34 smithi192 ceph-mon[25697]: [02/Apr/2024:05:10:33] ENGINE Bus STARTING 2024-04-02T05:10:34.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:34 smithi192 ceph-mon[25697]: mgrmap e37: y(active, since 2s), standbys: x 2024-04-02T05:10:34.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:34 smithi192 ceph-mon[25697]: [02/Apr/2024:05:10:33] ENGINE Serving on http://172.21.15.86:8765 2024-04-02T05:10:34.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:34 smithi192 ceph-mon[25697]: from='client.25363 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:10:34.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:34 smithi192 ceph-mon[25697]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:10:34.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:34 smithi192 ceph-mon[25697]: [02/Apr/2024:05:10:33] ENGINE Serving on https://172.21.15.86:7150 2024-04-02T05:10:34.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:34 smithi192 ceph-mon[25697]: [02/Apr/2024:05:10:33] ENGINE Bus STARTED 2024-04-02T05:10:35.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:34 smithi086 ceph-mon[29505]: [02/Apr/2024:05:10:33] ENGINE Bus STARTING 2024-04-02T05:10:35.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:34 smithi086 ceph-mon[29505]: mgrmap e37: y(active, since 2s), standbys: x 2024-04-02T05:10:35.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:34 smithi086 ceph-mon[29505]: [02/Apr/2024:05:10:33] ENGINE Serving on http://172.21.15.86:8765 2024-04-02T05:10:35.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:34 smithi086 ceph-mon[29505]: from='client.25363 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:10:35.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:34 smithi086 ceph-mon[29505]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:10:35.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:34 smithi086 ceph-mon[29505]: [02/Apr/2024:05:10:33] ENGINE Serving on https://172.21.15.86:7150 2024-04-02T05:10:35.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:34 smithi086 ceph-mon[29505]: [02/Apr/2024:05:10:33] ENGINE Bus STARTED 2024-04-02T05:10:35.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:34 smithi086 ceph-mon[24811]: [02/Apr/2024:05:10:33] ENGINE Bus STARTING 2024-04-02T05:10:35.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:34 smithi086 ceph-mon[24811]: mgrmap e37: y(active, since 2s), standbys: x 2024-04-02T05:10:35.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:34 smithi086 ceph-mon[24811]: [02/Apr/2024:05:10:33] ENGINE Serving on http://172.21.15.86:8765 2024-04-02T05:10:35.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:34 smithi086 ceph-mon[24811]: from='client.25363 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:10:35.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:34 smithi086 ceph-mon[24811]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:10:35.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:34 smithi086 ceph-mon[24811]: [02/Apr/2024:05:10:33] ENGINE Serving on https://172.21.15.86:7150 2024-04-02T05:10:35.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:34 smithi086 ceph-mon[24811]: [02/Apr/2024:05:10:33] ENGINE Bus STARTED 2024-04-02T05:10:35.192 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:10:35.192 INFO:teuthology.orchestra.run.smithi086.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", 2024-04-02T05:10:35.192 INFO:teuthology.orchestra.run.smithi086.stdout: "in_progress": true, 2024-04-02T05:10:35.192 INFO:teuthology.orchestra.run.smithi086.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-04-02T05:10:35.192 INFO:teuthology.orchestra.run.smithi086.stdout: "services_complete": [ 2024-04-02T05:10:35.192 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr" 2024-04-02T05:10:35.193 INFO:teuthology.orchestra.run.smithi086.stdout: ], 2024-04-02T05:10:35.193 INFO:teuthology.orchestra.run.smithi086.stdout: "progress": "2/2 daemons upgraded", 2024-04-02T05:10:35.193 INFO:teuthology.orchestra.run.smithi086.stdout: "message": "", 2024-04-02T05:10:35.193 INFO:teuthology.orchestra.run.smithi086.stdout: "is_paused": false 2024-04-02T05:10:35.193 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:10:35.263 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:34 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:10:34.962+0000 7f1bf509b200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-02T05:10:35.263 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:35 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:10:35.068+0000 7f1bf509b200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-02T05:10:35.263 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:35 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:10:35.260+0000 7f1bf509b200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-02T05:10:35.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:35 smithi086 ceph-mon[29505]: from='client.25369 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:10:35.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:35 smithi086 ceph-mon[29505]: from='client.25375 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:10:35.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:35 smithi086 ceph-mon[29505]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-02T05:10:35.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:35 smithi086 ceph-mon[29505]: Cluster is now healthy 2024-04-02T05:10:35.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:35 smithi086 ceph-mon[29505]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:10:35.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:35 smithi086 ceph-mon[29505]: mgrmap e38: y(active, since 3s), standbys: x 2024-04-02T05:10:35.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:35 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/788517322' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:10:35.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:35 smithi086 ceph-mon[24811]: from='client.25369 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:10:35.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:35 smithi086 ceph-mon[24811]: from='client.25375 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:10:35.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:35 smithi086 ceph-mon[24811]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-02T05:10:35.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:35 smithi086 ceph-mon[24811]: Cluster is now healthy 2024-04-02T05:10:35.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:35 smithi086 ceph-mon[24811]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:10:35.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:35 smithi086 ceph-mon[24811]: mgrmap e38: y(active, since 3s), standbys: x 2024-04-02T05:10:35.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:35 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/788517322' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:10:35.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:35 smithi192 ceph-mon[25697]: from='client.25369 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:10:35.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:35 smithi192 ceph-mon[25697]: from='client.25375 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:10:35.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:35 smithi192 ceph-mon[25697]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-02T05:10:35.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:35 smithi192 ceph-mon[25697]: Cluster is now healthy 2024-04-02T05:10:35.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:35 smithi192 ceph-mon[25697]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:10:35.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:35 smithi192 ceph-mon[25697]: mgrmap e38: y(active, since 3s), standbys: x 2024-04-02T05:10:35.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:35 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/788517322' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:10:36.388 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:36 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:10:36.253+0000 7f1bf509b200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-02T05:10:36.389 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:36 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:10:36.386+0000 7f1bf509b200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-02T05:10:37.056 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:36 smithi192 ceph-mon[25697]: from='client.25393 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:10:37.056 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:36 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:37.056 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:36 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:37.056 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:36 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:10:36.743+0000 7f1bf509b200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-02T05:10:37.056 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:36 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:10:36.839+0000 7f1bf509b200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-02T05:10:37.056 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:10:37.052+0000 7f1bf509b200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-02T05:10:37.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:36 smithi086 ceph-mon[24811]: from='client.25393 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:10:37.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:36 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:37.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:36 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:37.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:36 smithi086 ceph-mon[29505]: from='client.25393 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:10:37.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:36 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:37.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:36 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:37.427 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:10:37.251+0000 7f1bf509b200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-02T05:10:37.837 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:10:37.560+0000 7f1bf509b200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-02T05:10:37.837 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:10:37.833+0000 7f1bf509b200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-02T05:10:38.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:37 smithi086 ceph-mon[29505]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:10:38.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:37 smithi086 ceph-mon[29505]: mgrmap e39: y(active, since 5s), standbys: x 2024-04-02T05:10:38.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:37 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:38.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:37 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:38.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:37 smithi086 ceph-mon[24811]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:10:38.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:37 smithi086 ceph-mon[24811]: mgrmap e39: y(active, since 5s), standbys: x 2024-04-02T05:10:38.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:37 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:38.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:37 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:38.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:37 smithi192 ceph-mon[25697]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:10:38.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:37 smithi192 ceph-mon[25697]: mgrmap e39: y(active, since 5s), standbys: x 2024-04-02T05:10:38.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:37 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:38.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:37 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:38.177 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:38 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:10:38.079+0000 7f1bf509b200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-02T05:10:39.177 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:38 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:10:38.922+0000 7f1bf509b200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-02T05:10:39.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:38 smithi192 ceph-mon[25697]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:10:39.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:38 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:39.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:38 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:39.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:38 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:39.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:38 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:39.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:38 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:10:39.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:38 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:10:39.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:38 smithi086 ceph-mon[24811]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:10:39.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:38 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:39.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:38 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:39.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:38 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:39.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:38 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:39.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:38 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:10:39.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:38 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:10:39.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:38 smithi086 ceph-mon[29505]: from='client.25270 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:10:39.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:38 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:39.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:38 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:39.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:38 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:39.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:38 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:39.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:38 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:10:39.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:38 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:10:39.858 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:39 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:10:39.531+0000 7f1bf509b200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-02T05:10:39.858 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:39 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:10:39.633+0000 7f1bf509b200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-02T05:10:39.858 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:39 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:10:39.854+0000 7f1bf509b200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-02T05:10:40.177 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:39 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:05:10:39] ENGINE Bus STARTING 2024-04-02T05:10:40.177 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:39 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: CherryPy Checker: 2024-04-02T05:10:40.177 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:39 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: The Application mounted at '' has an empty config. 2024-04-02T05:10:40.177 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:39 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: 2024-04-02T05:10:40.177 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:39 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:05:10:39] ENGINE Serving on http://:::9283 2024-04-02T05:10:40.177 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:39 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[45425]: [02/Apr/2024:05:10:39] ENGINE Bus STARTED 2024-04-02T05:10:40.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:39 smithi192 ceph-mon[25697]: Updating smithi086:/etc/ceph/ceph.conf 2024-04-02T05:10:40.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:39 smithi192 ceph-mon[25697]: Updating smithi192:/etc/ceph/ceph.conf 2024-04-02T05:10:40.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:39 smithi192 ceph-mon[25697]: Updating smithi192:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T05:10:40.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:39 smithi192 ceph-mon[25697]: Updating smithi086:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T05:10:40.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:39 smithi192 ceph-mon[25697]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:10:40.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:39 smithi192 ceph-mon[25697]: Updating smithi192:/etc/ceph/ceph.client.admin.keyring 2024-04-02T05:10:40.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:39 smithi192 ceph-mon[25697]: Updating smithi086:/etc/ceph/ceph.client.admin.keyring 2024-04-02T05:10:40.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:39 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:40.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:39 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:40.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:39 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:40.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:39 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:40.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:39 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:40.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:39 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:10:40.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:39 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:10:40.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:39 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:10:40.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:39 smithi192 ceph-mon[25697]: Standby manager daemon x restarted 2024-04-02T05:10:40.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:39 smithi192 ceph-mon[25697]: Standby manager daemon x started 2024-04-02T05:10:40.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:39 smithi192 ceph-mon[25697]: from='mgr.? 172.21.15.192:0/275858538' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-02T05:10:40.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:39 smithi192 ceph-mon[25697]: from='mgr.? 172.21.15.192:0/275858538' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T05:10:40.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:39 smithi192 ceph-mon[25697]: from='mgr.? 172.21.15.192:0/275858538' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-02T05:10:40.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:39 smithi192 ceph-mon[25697]: from='mgr.? 172.21.15.192:0/275858538' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T05:10:40.187 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[29505]: Updating smithi086:/etc/ceph/ceph.conf 2024-04-02T05:10:40.187 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[29505]: Updating smithi192:/etc/ceph/ceph.conf 2024-04-02T05:10:40.187 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[29505]: Updating smithi192:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T05:10:40.187 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[29505]: Updating smithi086:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T05:10:40.187 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[29505]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:10:40.187 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[29505]: Updating smithi192:/etc/ceph/ceph.client.admin.keyring 2024-04-02T05:10:40.188 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[29505]: Updating smithi086:/etc/ceph/ceph.client.admin.keyring 2024-04-02T05:10:40.188 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:40.188 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:40.188 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:40.188 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:40.188 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:40.188 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:10:40.188 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:10:40.188 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:10:40.188 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[29505]: Standby manager daemon x restarted 2024-04-02T05:10:40.188 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[29505]: Standby manager daemon x started 2024-04-02T05:10:40.188 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.192:0/275858538' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-02T05:10:40.188 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.192:0/275858538' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T05:10:40.188 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.192:0/275858538' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-02T05:10:40.189 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.192:0/275858538' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T05:10:40.189 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[24811]: Updating smithi086:/etc/ceph/ceph.conf 2024-04-02T05:10:40.189 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[24811]: Updating smithi192:/etc/ceph/ceph.conf 2024-04-02T05:10:40.189 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[24811]: Updating smithi192:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T05:10:40.189 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[24811]: Updating smithi086:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T05:10:40.189 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[24811]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:10:40.189 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[24811]: Updating smithi192:/etc/ceph/ceph.client.admin.keyring 2024-04-02T05:10:40.189 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[24811]: Updating smithi086:/etc/ceph/ceph.client.admin.keyring 2024-04-02T05:10:40.189 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:40.189 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:40.189 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:40.189 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:40.189 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:40.190 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:10:40.190 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:10:40.190 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:10:40.190 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[24811]: Standby manager daemon x restarted 2024-04-02T05:10:40.190 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[24811]: Standby manager daemon x started 2024-04-02T05:10:40.190 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[24811]: from='mgr.? 172.21.15.192:0/275858538' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-02T05:10:40.190 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[24811]: from='mgr.? 172.21.15.192:0/275858538' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T05:10:40.190 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[24811]: from='mgr.? 172.21.15.192:0/275858538' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-02T05:10:40.190 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:39 smithi086 ceph-mon[24811]: from='mgr.? 172.21.15.192:0/275858538' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T05:10:41.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:40 smithi086 ceph-mon[24811]: Updating smithi086:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-02T05:10:41.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:40 smithi086 ceph-mon[24811]: Updating smithi192:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-02T05:10:41.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:40 smithi086 ceph-mon[24811]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 21 op/s 2024-04-02T05:10:41.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:40 smithi086 ceph-mon[24811]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T05:10:41.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:40 smithi086 ceph-mon[24811]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T05:10:41.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:40 smithi086 ceph-mon[24811]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-02T05:10:41.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:40 smithi086 ceph-mon[24811]: mgrmap e40: y(active, since 8s), standbys: x 2024-04-02T05:10:41.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:40 smithi086 ceph-mon[29505]: Updating smithi086:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-02T05:10:41.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:40 smithi086 ceph-mon[29505]: Updating smithi192:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-02T05:10:41.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:40 smithi086 ceph-mon[29505]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 21 op/s 2024-04-02T05:10:41.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:40 smithi086 ceph-mon[29505]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T05:10:41.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:40 smithi086 ceph-mon[29505]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T05:10:41.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:40 smithi086 ceph-mon[29505]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-02T05:10:41.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:40 smithi086 ceph-mon[29505]: mgrmap e40: y(active, since 8s), standbys: x 2024-04-02T05:10:41.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:40 smithi192 ceph-mon[25697]: Updating smithi086:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-02T05:10:41.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:40 smithi192 ceph-mon[25697]: Updating smithi192:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-02T05:10:41.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:40 smithi192 ceph-mon[25697]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 21 op/s 2024-04-02T05:10:41.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:40 smithi192 ceph-mon[25697]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T05:10:41.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:40 smithi192 ceph-mon[25697]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T05:10:41.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:40 smithi192 ceph-mon[25697]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-02T05:10:41.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:40 smithi192 ceph-mon[25697]: mgrmap e40: y(active, since 8s), standbys: x 2024-04-02T05:10:41.710 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:10:41 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[59010]: level=info ts=2024-04-02T05:10:41.596Z caller=notify.go:735 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify success" attempts=6 2024-04-02T05:10:42.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:10:42.494+0000 7f180812e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:10:42.834 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Creating ceph-iscsi config... 2024-04-02T05:10:42.834 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:10:42.834 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:10:42.834 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:10:42.834 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:10:42.834 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:10:42.834 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Traceback (most recent call last): 2024-04-02T05:10:42.834 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:10:42.834 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: return _run_code(code, main_globals, None, 2024-04-02T05:10:42.834 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:10:42.834 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: exec(code, run_globals) 2024-04-02T05:10:42.834 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:10:42.834 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:10:42.835 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:10:42.835 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:10:42.835 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:10:42.835 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:10:42.835 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:10:42.835 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:10:42.836 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:10:42.836 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Traceback (most recent call last): 2024-04-02T05:10:42.836 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T05:10:42.836 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T05:10:42.836 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T05:10:42.836 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T05:10:42.836 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T05:10:42.836 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: return self.event_loop.get_result(coro, timeout) 2024-04-02T05:10:42.836 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T05:10:42.836 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: return future.result(timeout) 2024-04-02T05:10:42.836 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T05:10:42.836 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: return self.__get_result() 2024-04-02T05:10:42.836 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T05:10:42.837 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: raise self._exception 2024-04-02T05:10:42.837 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T05:10:42.837 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: config_blobs=daemon_spec.final_config, 2024-04-02T05:10:42.837 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T05:10:42.837 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T05:10:42.837 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:10:42.837 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Creating ceph-iscsi config... 2024-04-02T05:10:42.837 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:10:42.837 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:10:42.837 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:10:42.837 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:10:42.837 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:10:42.837 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Traceback (most recent call last): 2024-04-02T05:10:42.838 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:10:42.838 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: return _run_code(code, main_globals, None, 2024-04-02T05:10:42.838 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:10:42.838 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: exec(code, run_globals) 2024-04-02T05:10:42.838 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:10:42.838 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:10:42.838 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:10:42.838 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:10:42.838 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:10:42.838 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:10:42.838 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:10:42.838 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:10:42.838 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:10:43.280 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:42 smithi192 ceph-mon[25697]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 0 B/s wr, 15 op/s 2024-04-02T05:10:43.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-mon[29505]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 0 B/s wr, 15 op/s 2024-04-02T05:10:43.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:42 smithi086 ceph-mon[24811]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 0 B/s wr, 15 op/s 2024-04-02T05:10:44.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:43 smithi086 ceph-mon[29505]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:10:44.333 INFO:journalctl@ceph.mon.c.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T05:10:44.333 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:10:44.333 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:10:44.333 INFO:journalctl@ceph.mon.c.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:10:44.333 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:10:44.334 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:10:44.334 INFO:journalctl@ceph.mon.c.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:10:44.334 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:10:44.334 INFO:journalctl@ceph.mon.c.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:10:44.334 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:10:44.334 INFO:journalctl@ceph.mon.c.smithi086.stdout: exec(code, run_globals) 2024-04-02T05:10:44.334 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:10:44.334 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:10:44.334 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:10:44.334 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:10:44.334 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:10:44.334 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:10:44.334 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:10:44.335 INFO:journalctl@ceph.mon.c.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:10:44.335 INFO:journalctl@ceph.mon.c.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:10:44.335 INFO:journalctl@ceph.mon.c.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:10:44.335 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T05:10:44.335 INFO:journalctl@ceph.mon.c.smithi086.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T05:10:44.335 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T05:10:44.335 INFO:journalctl@ceph.mon.c.smithi086.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T05:10:44.335 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T05:10:44.335 INFO:journalctl@ceph.mon.c.smithi086.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-02T05:10:44.335 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T05:10:44.335 INFO:journalctl@ceph.mon.c.smithi086.stdout: return future.result(timeout) 2024-04-02T05:10:44.335 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T05:10:44.335 INFO:journalctl@ceph.mon.c.smithi086.stdout: return self.__get_result() 2024-04-02T05:10:44.335 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T05:10:44.336 INFO:journalctl@ceph.mon.c.smithi086.stdout: raise self._exception 2024-04-02T05:10:44.336 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T05:10:44.336 INFO:journalctl@ceph.mon.c.smithi086.stdout: config_blobs=daemon_spec.final_config, 2024-04-02T05:10:44.336 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T05:10:44.336 INFO:journalctl@ceph.mon.c.smithi086.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T05:10:44.336 INFO:journalctl@ceph.mon.c.smithi086.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:10:44.336 INFO:journalctl@ceph.mon.c.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T05:10:44.336 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:10:44.336 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:10:44.336 INFO:journalctl@ceph.mon.c.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:10:44.336 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:10:44.336 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:10:44.336 INFO:journalctl@ceph.mon.c.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:10:44.336 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:10:44.337 INFO:journalctl@ceph.mon.c.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:10:44.337 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:10:44.337 INFO:journalctl@ceph.mon.c.smithi086.stdout: exec(code, run_globals) 2024-04-02T05:10:44.337 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:10:44.337 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:10:44.337 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:10:44.337 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:10:44.337 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:43 smithi086 ceph-mon[24811]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:10:44.337 INFO:journalctl@ceph.mon.a.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T05:10:44.337 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:10:44.337 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:10:44.338 INFO:journalctl@ceph.mon.a.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:10:44.338 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:10:44.338 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:10:44.338 INFO:journalctl@ceph.mon.a.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:10:44.338 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:10:44.338 INFO:journalctl@ceph.mon.a.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:10:44.338 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:10:44.338 INFO:journalctl@ceph.mon.a.smithi086.stdout: exec(code, run_globals) 2024-04-02T05:10:44.338 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:10:44.338 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:10:44.338 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:10:44.338 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:10:44.339 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:10:44.339 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:10:44.339 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:10:44.339 INFO:journalctl@ceph.mon.a.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:10:44.339 INFO:journalctl@ceph.mon.a.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:10:44.339 INFO:journalctl@ceph.mon.a.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:10:44.339 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T05:10:44.339 INFO:journalctl@ceph.mon.a.smithi086.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T05:10:44.339 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T05:10:44.339 INFO:journalctl@ceph.mon.a.smithi086.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T05:10:44.339 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T05:10:44.339 INFO:journalctl@ceph.mon.a.smithi086.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-02T05:10:44.339 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T05:10:44.339 INFO:journalctl@ceph.mon.a.smithi086.stdout: return future.result(timeout) 2024-04-02T05:10:44.340 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T05:10:44.340 INFO:journalctl@ceph.mon.a.smithi086.stdout: return self.__get_result() 2024-04-02T05:10:44.340 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T05:10:44.340 INFO:journalctl@ceph.mon.a.smithi086.stdout: raise self._exception 2024-04-02T05:10:44.340 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T05:10:44.340 INFO:journalctl@ceph.mon.a.smithi086.stdout: config_blobs=daemon_spec.final_config, 2024-04-02T05:10:44.340 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T05:10:44.340 INFO:journalctl@ceph.mon.a.smithi086.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T05:10:44.340 INFO:journalctl@ceph.mon.a.smithi086.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:10:44.340 INFO:journalctl@ceph.mon.a.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T05:10:44.340 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:10:44.340 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:10:44.340 INFO:journalctl@ceph.mon.a.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:10:44.340 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:10:44.340 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:10:44.341 INFO:journalctl@ceph.mon.a.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:10:44.341 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:10:44.341 INFO:journalctl@ceph.mon.a.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:10:44.341 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:10:44.341 INFO:journalctl@ceph.mon.a.smithi086.stdout: exec(code, run_globals) 2024-04-02T05:10:44.341 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:10:44.341 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:10:44.341 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:10:44.341 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:10:44.341 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:10:44.341 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:10:44.341 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:10:44.341 INFO:journalctl@ceph.mon.a.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:10:44.342 INFO:journalctl@ceph.mon.a.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:10:44.342 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:43 smithi086 ceph-mon[24811]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-02T05:10:44.342 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:43 smithi086 ceph-mon[24811]: Reconfiguring daemon prometheus.a on smithi192 2024-04-02T05:10:44.342 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:10:44.342 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:10:44.342 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:10:44.342 INFO:journalctl@ceph.mon.c.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:10:44.342 INFO:journalctl@ceph.mon.c.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:10:44.342 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:43 smithi086 ceph-mon[29505]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-02T05:10:44.342 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:43 smithi086 ceph-mon[29505]: Reconfiguring daemon prometheus.a on smithi192 2024-04-02T05:10:44.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:43 smithi192 ceph-mon[25697]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:10:44.427 INFO:journalctl@ceph.mon.b.smithi192.stdout: Creating ceph-iscsi config... 2024-04-02T05:10:44.427 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:10:44.427 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:10:44.427 INFO:journalctl@ceph.mon.b.smithi192.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:10:44.427 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:10:44.428 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:10:44.428 INFO:journalctl@ceph.mon.b.smithi192.stdout: Traceback (most recent call last): 2024-04-02T05:10:44.428 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:10:44.428 INFO:journalctl@ceph.mon.b.smithi192.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:10:44.428 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:10:44.428 INFO:journalctl@ceph.mon.b.smithi192.stdout: exec(code, run_globals) 2024-04-02T05:10:44.428 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:10:44.428 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:10:44.428 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:10:44.428 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:10:44.428 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:10:44.428 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:10:44.428 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:10:44.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:10:44.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:10:44.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: Traceback (most recent call last): 2024-04-02T05:10:44.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T05:10:44.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T05:10:44.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T05:10:44.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T05:10:44.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T05:10:44.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-02T05:10:44.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T05:10:44.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: return future.result(timeout) 2024-04-02T05:10:44.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T05:10:44.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: return self.__get_result() 2024-04-02T05:10:44.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T05:10:44.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: raise self._exception 2024-04-02T05:10:44.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T05:10:44.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: config_blobs=daemon_spec.final_config, 2024-04-02T05:10:44.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T05:10:44.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T05:10:44.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:10:44.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: Creating ceph-iscsi config... 2024-04-02T05:10:44.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:10:44.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:10:44.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:10:44.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:10:44.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:10:44.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: Traceback (most recent call last): 2024-04-02T05:10:44.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:10:44.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:10:44.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:10:44.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: exec(code, run_globals) 2024-04-02T05:10:44.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:10:44.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:10:44.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:10:44.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:10:44.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:10:44.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:10:44.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:10:44.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:10:44.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:10:44.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:43 smithi192 ceph-mon[25697]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-02T05:10:44.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:43 smithi192 ceph-mon[25697]: Reconfiguring daemon prometheus.a on smithi192 2024-04-02T05:10:45.041 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:44 smithi192 ceph-mon[25697]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-02T05:10:45.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:44 smithi086 ceph-mon[29505]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-02T05:10:45.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:44 smithi086 ceph-mon[24811]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-02T05:10:46.162 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:46 smithi192 systemd[1]: Stopping Ceph prometheus.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:10:46.162 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:46 smithi192 bash[60733]: Error: no container with name or ID "ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus.a" found: no such container 2024-04-02T05:10:46.427 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:46 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:10:46.159Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-02T05:10:46.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:46 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:10:46.160Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-02T05:10:46.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:46 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:10:46.160Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-02T05:10:46.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:46 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:10:46.160Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-02T05:10:46.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:46 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:10:46.160Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-02T05:10:46.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:46 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:10:46.160Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-02T05:10:46.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:46 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:10:46.160Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-02T05:10:46.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:46 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:10:46.160Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-02T05:10:46.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:46 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:10:46.160Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-02T05:10:46.429 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:46 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:10:46.160Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-02T05:10:46.429 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:46 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:10:46.160Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-02T05:10:46.429 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:46 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[58029]: ts=2024-04-02T05:10:46.160Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-02T05:10:46.429 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:46 smithi192 podman[60745]: 2024-04-02 05:10:46.170399165 +0000 UTC m=+0.126165362 container died f455b2541990855818e04134d01876b2840e7ee552a43cb652163b1ac3d40d8b (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T05:10:46.713 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:46 smithi192 podman[60745]: 2024-04-02 05:10:46.484188751 +0000 UTC m=+0.439954929 container cleanup f455b2541990855818e04134d01876b2840e7ee552a43cb652163b1ac3d40d8b (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T05:10:46.713 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:46 smithi192 bash[60745]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a 2024-04-02T05:10:46.713 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:46 smithi192 bash[60774]: Error: no container with name or ID "ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus.a" found: no such container 2024-04-02T05:10:47.112 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:46 smithi192 ceph-mon[25697]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-02T05:10:47.113 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:46 smithi192 podman[60759]: 2024-04-02 05:10:46.711153893 +0000 UTC m=+0.539806520 container remove f455b2541990855818e04134d01876b2840e7ee552a43cb652163b1ac3d40d8b (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T05:10:47.113 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:46 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@prometheus.a.service: Deactivated successfully. 2024-04-02T05:10:47.113 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:46 smithi192 systemd[1]: Stopped Ceph prometheus.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:10:47.113 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:46 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@prometheus.a.service: Consumed 1.312s CPU time. 2024-04-02T05:10:47.113 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:46 smithi192 systemd[1]: Starting Ceph prometheus.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:10:47.113 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:46 smithi192 podman[60844]: 2024-04-02 05:10:46.855134012 +0000 UTC m=+0.019251445 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-02T05:10:47.113 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:46 smithi192 podman[60844]: 2024-04-02 05:10:46.95401481 +0000 UTC m=+0.118132224 container create f61f481e74750b5833c61a0e6a9ba4dc8ce0bcb5407d0e1203fe9a7c6d4eedd5 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T05:10:47.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:46 smithi086 ceph-mon[29505]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-02T05:10:47.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:46 smithi086 ceph-mon[24811]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-02T05:10:47.427 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:47 smithi192 podman[60844]: 2024-04-02 05:10:47.240280514 +0000 UTC m=+0.404397929 container init f61f481e74750b5833c61a0e6a9ba4dc8ce0bcb5407d0e1203fe9a7c6d4eedd5 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T05:10:47.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:47 smithi192 podman[60844]: 2024-04-02 05:10:47.243357711 +0000 UTC m=+0.407475137 container start f61f481e74750b5833c61a0e6a9ba4dc8ce0bcb5407d0e1203fe9a7c6d4eedd5 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T05:10:47.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[60859]: ts=2024-04-02T05:10:47.271Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-02T05:10:47.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[60859]: ts=2024-04-02T05:10:47.271Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-02T05:10:47.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[60859]: ts=2024-04-02T05:10:47.271Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-02T05:10:47.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[60859]: ts=2024-04-02T05:10:47.271Z caller=main.go:518 level=info host_details="(Linux 5.14.0-434.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Mar 26 19:20:47 UTC 2024 x86_64 smithi192 (none))" 2024-04-02T05:10:47.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[60859]: ts=2024-04-02T05:10:47.271Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-02T05:10:47.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[60859]: ts=2024-04-02T05:10:47.271Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-02T05:10:47.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[60859]: ts=2024-04-02T05:10:47.273Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-02T05:10:47.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[60859]: ts=2024-04-02T05:10:47.273Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-02T05:10:47.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[60859]: ts=2024-04-02T05:10:47.274Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-02T05:10:47.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[60859]: ts=2024-04-02T05:10:47.275Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-02T05:10:47.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[60859]: ts=2024-04-02T05:10:47.275Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=3.519µs 2024-04-02T05:10:47.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[60859]: ts=2024-04-02T05:10:47.275Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-02T05:10:47.428 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:47 smithi192 bash[60844]: f61f481e74750b5833c61a0e6a9ba4dc8ce0bcb5407d0e1203fe9a7c6d4eedd5 2024-04-02T05:10:47.429 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:47 smithi192 systemd[1]: Started Ceph prometheus.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:10:47.739 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[60859]: ts=2024-04-02T05:10:47.480Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=5 2024-04-02T05:10:47.739 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[60859]: ts=2024-04-02T05:10:47.737Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=5 2024-04-02T05:10:48.002 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[60859]: ts=2024-04-02T05:10:47.788Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=5 2024-04-02T05:10:48.002 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[60859]: ts=2024-04-02T05:10:47.836Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=5 2024-04-02T05:10:48.002 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[60859]: ts=2024-04-02T05:10:47.840Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=5 2024-04-02T05:10:48.002 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[60859]: ts=2024-04-02T05:10:47.840Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=5 2024-04-02T05:10:48.002 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[60859]: ts=2024-04-02T05:10:47.840Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=53.287µs wal_replay_duration=564.864182ms total_replay_duration=564.940541ms 2024-04-02T05:10:48.003 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[60859]: ts=2024-04-02T05:10:47.844Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-02T05:10:48.003 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[60859]: ts=2024-04-02T05:10:47.844Z caller=main.go:947 level=info msg="TSDB started" 2024-04-02T05:10:48.003 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[60859]: ts=2024-04-02T05:10:47.844Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-02T05:10:48.003 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[60859]: ts=2024-04-02T05:10:47.867Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=22.506891ms db_storage=899ns remote_storage=1.244µs web_handler=407ns query_engine=792ns scrape=341.237µs scrape_sd=71.061µs notify=14.133µs notify_sd=18.01µs rules=21.828087ms 2024-04-02T05:10:48.003 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:10:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[60859]: ts=2024-04-02T05:10:47.867Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-02T05:10:48.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:48 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:48.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:48 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:48.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:48 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T05:10:48.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:48 smithi192 ceph-mon[25697]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T05:10:48.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:48 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:10:48.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:48 smithi192 ceph-mon[25697]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-02T05:10:48.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:48 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:10:48.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:48 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:48.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:48 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:48.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:48 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T05:10:48.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:48 smithi086 ceph-mon[29505]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T05:10:48.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:48 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:10:48.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:48 smithi086 ceph-mon[29505]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-02T05:10:48.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:48 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:10:48.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:48 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:48.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:48 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:48.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:48 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T05:10:48.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:48 smithi086 ceph-mon[24811]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T05:10:48.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:48 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:10:48.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:48 smithi086 ceph-mon[24811]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-02T05:10:48.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:48 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:10:49.595 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:49 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T05:10:49.595 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:49 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T05:10:49.595 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:49 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T05:10:49.595 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:49 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:10:49.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:49 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T05:10:49.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:49 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T05:10:49.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:49 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T05:10:49.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:49 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:10:49.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:49 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T05:10:49.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:49 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T05:10:49.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:49 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T05:10:49.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:49 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:10:50.641 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:50 smithi192 ceph-mon[25697]: Upgrade: Updating mgr.x 2024-04-02T05:10:50.641 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:50 smithi192 ceph-mon[25697]: Deploying daemon mgr.x on smithi192 2024-04-02T05:10:50.641 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:50 smithi192 ceph-mon[25697]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 100 KiB/s rd, 0 B/s wr, 144 op/s 2024-04-02T05:10:50.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:50 smithi086 ceph-mon[29505]: Upgrade: Updating mgr.x 2024-04-02T05:10:50.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:50 smithi086 ceph-mon[29505]: Deploying daemon mgr.x on smithi192 2024-04-02T05:10:50.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:50 smithi086 ceph-mon[29505]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 100 KiB/s rd, 0 B/s wr, 144 op/s 2024-04-02T05:10:50.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:50 smithi086 ceph-mon[24811]: Upgrade: Updating mgr.x 2024-04-02T05:10:50.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:50 smithi086 ceph-mon[24811]: Deploying daemon mgr.x on smithi192 2024-04-02T05:10:50.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:50 smithi086 ceph-mon[24811]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 100 KiB/s rd, 0 B/s wr, 144 op/s 2024-04-02T05:10:50.927 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:50 smithi192 systemd[1]: Stopping Ceph mgr.x for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:10:51.397 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:51 smithi192 podman[61200]: 2024-04-02 05:10:51.108474436 +0000 UTC m=+0.128330487 container died fd618149d6b41b92546c4791f5c712a6d3d78106a1d99348c3be723483b74eaa (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-6cec793, io.buildah.version=1.33.5) 2024-04-02T05:10:51.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:51 smithi192 podman[61200]: 2024-04-02 05:10:51.48610947 +0000 UTC m=+0.505965522 container cleanup fd618149d6b41b92546c4791f5c712a6d3d78106a1d99348c3be723483b74eaa (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=squid-6cec793, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0) 2024-04-02T05:10:51.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:51 smithi192 bash[61200]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x 2024-04-02T05:10:52.100 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:51 smithi192 podman[61214]: 2024-04-02 05:10:51.7623407 +0000 UTC m=+0.657210630 container remove fd618149d6b41b92546c4791f5c712a6d3d78106a1d99348c3be723483b74eaa (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x, org.label-schema.vendor=CentOS, RELEASE=squid-6cec793, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-04-02T05:10:52.100 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:51 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-02T05:10:52.100 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:51 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mgr.x.service: Failed with result 'exit-code'. 2024-04-02T05:10:52.100 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:51 smithi192 systemd[1]: Stopped Ceph mgr.x for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:10:52.100 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:51 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mgr.x.service: Consumed 58.744s CPU time. 2024-04-02T05:10:52.427 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:52 smithi192 systemd[1]: Starting Ceph mgr.x for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:10:52.427 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:52 smithi192 podman[61349]: 2024-04-02 05:10:52.214580861 +0000 UTC m=+0.018269946 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:10:52.427 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:52 smithi192 podman[61349]: 2024-04-02 05:10:52.322331606 +0000 UTC m=+0.126020726 container create dd1e6d0f0a95fe679c7b415b6610151be8c3934405421ba58a2ffec5cc2cb32b (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=squid-6cec793) 2024-04-02T05:10:52.925 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:52 smithi192 podman[61349]: 2024-04-02 05:10:52.524706213 +0000 UTC m=+0.328395288 container init dd1e6d0f0a95fe679c7b415b6610151be8c3934405421ba58a2ffec5cc2cb32b (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x, RELEASE=squid-6cec793, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-04-02T05:10:52.925 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:52 smithi192 podman[61349]: 2024-04-02 05:10:52.527151167 +0000 UTC m=+0.330840246 container start dd1e6d0f0a95fe679c7b415b6610151be8c3934405421ba58a2ffec5cc2cb32b (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-6cec793, org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-02T05:10:52.925 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:52 smithi192 ceph-mgr[61368]: -- 172.21.15.192:0/1474150711 <== mon.2 v2:172.21.15.192:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55e5ded7e5a0 con 0x55e5ded53000 2024-04-02T05:10:52.925 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:52 smithi192 bash[61349]: dd1e6d0f0a95fe679c7b415b6610151be8c3934405421ba58a2ffec5cc2cb32b 2024-04-02T05:10:52.925 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:52 smithi192 systemd[1]: Started Ceph mgr.x for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:10:53.172 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:52 smithi086 ceph-mon[29505]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-02T05:10:53.172 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:52 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:53.172 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:52 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:53.172 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:52 smithi086 ceph-mon[24811]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-02T05:10:53.172 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:52 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:53.172 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:52 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:53.177 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:53 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:10:53.100+0000 7f98fedc8200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-02T05:10:53.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:52 smithi192 ceph-mon[25697]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-02T05:10:53.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:52 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:53.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:52 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:53.454 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:10:53 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:10:53] "GET /metrics HTTP/1.1" 200 37988 "" "Prometheus/2.33.4" 2024-04-02T05:10:53.499 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:53 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:10:53.210+0000 7f98fedc8200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-02T05:10:54.152 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:53 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:10:53.890+0000 7f98fedc8200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-02T05:10:54.152 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:54 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:10:54.002+0000 7f98fedc8200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-02T05:10:54.427 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:54 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:10:54.149+0000 7f98fedc8200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-02T05:10:54.427 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:54 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:10:54.246+0000 7f98fedc8200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-02T05:10:54.427 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:54 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:10:54.334+0000 7f98fedc8200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-02T05:10:54.734 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:54 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:10:54.607+0000 7f98fedc8200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-02T05:10:55.142 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:54 smithi192 ceph-mon[25697]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-02T05:10:55.143 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:54 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/116295888' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T05:10:55.143 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:54 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:10:54.731+0000 7f98fedc8200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-02T05:10:55.143 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:54 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:10:54.836+0000 7f98fedc8200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-02T05:10:55.143 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:54 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:10:54.959+0000 7f98fedc8200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-02T05:10:55.143 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:55 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:10:55.140+0000 7f98fedc8200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-02T05:10:55.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:54 smithi086 ceph-mon[24811]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-02T05:10:55.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:54 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/116295888' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T05:10:55.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:54 smithi086 ceph-mon[29505]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-02T05:10:55.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:54 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/116295888' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T05:10:56.427 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:56 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:10:56.167+0000 7f98fedc8200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-02T05:10:56.427 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:56 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:10:56.299+0000 7f98fedc8200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-02T05:10:57.009 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:56 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:10:56.694+0000 7f98fedc8200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-02T05:10:57.009 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:56 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:10:56.792+0000 7f98fedc8200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-02T05:10:57.010 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:56 smithi192 ceph-mon[25697]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-02T05:10:57.010 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:56 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:57.010 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:56 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:57.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:56 smithi086 ceph-mon[24811]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-02T05:10:57.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:56 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:57.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:56 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:57.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:56 smithi086 ceph-mon[29505]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-02T05:10:57.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:56 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:57.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:56 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:57.427 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:10:57.006+0000 7f98fedc8200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-02T05:10:57.427 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:10:57.207+0000 7f98fedc8200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-02T05:10:57.801 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:10:57.518+0000 7f98fedc8200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-02T05:10:58.059 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:10:57.799+0000 7f98fedc8200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-02T05:10:58.417 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:58 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:10:58.056+0000 7f98fedc8200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-02T05:10:58.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:58 smithi192 ceph-mon[25697]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 71 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-02T05:10:58.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:58 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:58.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:10:58 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:59.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:58 smithi086 ceph-mon[29505]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 71 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-02T05:10:59.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:58 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:59.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:10:58 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:59.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:58 smithi086 ceph-mon[24811]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 71 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-02T05:10:59.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:58 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:59.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:10:58 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:10:59.386 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:58 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:10:58.930+0000 7f98fedc8200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-02T05:10:59.640 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:59 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:10:59.524+0000 7f98fedc8200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-02T05:10:59.640 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:59 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:10:59.632+0000 7f98fedc8200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-02T05:10:59.927 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:59 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:10:59.860+0000 7f98fedc8200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-02T05:10:59.927 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:59 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: [02/Apr/2024:05:10:59] ENGINE Bus STARTING 2024-04-02T05:11:00.427 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:59 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: CherryPy Checker: 2024-04-02T05:11:00.427 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:59 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: The Application mounted at '' has an empty config. 2024-04-02T05:11:00.427 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:59 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:11:00.427 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:59 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: [02/Apr/2024:05:10:59] ENGINE Serving on http://:::9283 2024-04-02T05:11:00.427 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:10:59 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: [02/Apr/2024:05:10:59] ENGINE Bus STARTED 2024-04-02T05:11:00.955 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:00 smithi192 ceph-mon[25697]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-02T05:11:00.955 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:00 smithi192 ceph-mon[25697]: Standby manager daemon x restarted 2024-04-02T05:11:00.956 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:00 smithi192 ceph-mon[25697]: Standby manager daemon x started 2024-04-02T05:11:00.956 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:00 smithi192 ceph-mon[25697]: from='mgr.? 172.21.15.192:0/3504804197' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-02T05:11:00.956 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:00 smithi192 ceph-mon[25697]: from='mgr.? 172.21.15.192:0/3504804197' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T05:11:00.956 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:00 smithi192 ceph-mon[25697]: from='mgr.? 172.21.15.192:0/3504804197' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-02T05:11:00.956 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:00 smithi192 ceph-mon[25697]: from='mgr.? 172.21.15.192:0/3504804197' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T05:11:01.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:00 smithi086 ceph-mon[29505]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-02T05:11:01.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:00 smithi086 ceph-mon[29505]: Standby manager daemon x restarted 2024-04-02T05:11:01.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:00 smithi086 ceph-mon[29505]: Standby manager daemon x started 2024-04-02T05:11:01.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:00 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.192:0/3504804197' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-02T05:11:01.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:00 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.192:0/3504804197' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T05:11:01.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:00 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.192:0/3504804197' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-02T05:11:01.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:00 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.192:0/3504804197' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T05:11:01.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:00 smithi086 ceph-mon[24811]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-02T05:11:01.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:00 smithi086 ceph-mon[24811]: Standby manager daemon x restarted 2024-04-02T05:11:01.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:00 smithi086 ceph-mon[24811]: Standby manager daemon x started 2024-04-02T05:11:01.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:00 smithi086 ceph-mon[24811]: from='mgr.? 172.21.15.192:0/3504804197' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-02T05:11:01.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:00 smithi086 ceph-mon[24811]: from='mgr.? 172.21.15.192:0/3504804197' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T05:11:01.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:00 smithi086 ceph-mon[24811]: from='mgr.? 172.21.15.192:0/3504804197' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-02T05:11:01.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:00 smithi086 ceph-mon[24811]: from='mgr.? 172.21.15.192:0/3504804197' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T05:11:02.079 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:01 smithi086 ceph-mon[24811]: mgrmap e41: y(active, since 29s), standbys: x 2024-04-02T05:11:02.079 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:01 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:02.079 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:01 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:02.079 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:01 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:11:02.079 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:01 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:11:02.079 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:01 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:02.079 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:01 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:11:02.079 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:01 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:11:02.079 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:01 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:11:02.080 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:01 smithi086 ceph-mon[29505]: mgrmap e41: y(active, since 29s), standbys: x 2024-04-02T05:11:02.080 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:01 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:02.080 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:01 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:02.080 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:01 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:11:02.080 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:01 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:11:02.080 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:01 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:02.080 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:01 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:11:02.080 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:01 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:11:02.080 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:01 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:11:02.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:01 smithi192 ceph-mon[25697]: mgrmap e41: y(active, since 29s), standbys: x 2024-04-02T05:11:02.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:01 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:02.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:01 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:02.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:01 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:11:02.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:01 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:11:02.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:01 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:02.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:01 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:11:02.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:01 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:11:02.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:01 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:11:03.081 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:02 smithi086 ceph-mon[29505]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 429 B/s rd, 0 op/s 2024-04-02T05:11:03.082 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:02 smithi086 ceph-mon[29505]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 515 B/s rd, 0 op/s 2024-04-02T05:11:03.082 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:02 smithi086 ceph-mon[29505]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T05:11:03.082 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:02 smithi086 ceph-mon[29505]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T05:11:03.082 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:02 smithi086 ceph-mon[24811]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 429 B/s rd, 0 op/s 2024-04-02T05:11:03.082 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:02 smithi086 ceph-mon[24811]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 515 B/s rd, 0 op/s 2024-04-02T05:11:03.082 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:02 smithi086 ceph-mon[24811]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T05:11:03.082 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:02 smithi086 ceph-mon[24811]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T05:11:03.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:02 smithi192 ceph-mon[25697]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 429 B/s rd, 0 op/s 2024-04-02T05:11:03.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:02 smithi192 ceph-mon[25697]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 515 B/s rd, 0 op/s 2024-04-02T05:11:03.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:02 smithi192 ceph-mon[25697]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T05:11:03.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:02 smithi192 ceph-mon[25697]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T05:11:03.797 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:11:03] "GET /metrics HTTP/1.1" 200 37986 "" "Prometheus/2.33.4" 2024-04-02T05:11:04.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:03 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:11:04.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:03 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:11:04.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:03 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:11:05.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[29505]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 515 B/s rd, 0 op/s 2024-04-02T05:11:05.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:11:05.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:05.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:05.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-02T05:11:05.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-02T05:11:05.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-02T05:11:05.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-02T05:11:05.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-02T05:11:05.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-02T05:11:05.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:05.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:05.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:05.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:05.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:05.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:05.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:05.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:05.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:05.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:05.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:05.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:05.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:05.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:05.085 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:11:04.632+0000 7f180812e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:11:05.085 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Creating ceph-iscsi config... 2024-04-02T05:11:05.085 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:11:05.085 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:11:05.085 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:11:05.085 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:05.085 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:05.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Traceback (most recent call last): 2024-04-02T05:11:05.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:11:05.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: return _run_code(code, main_globals, None, 2024-04-02T05:11:05.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:11:05.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: exec(code, run_globals) 2024-04-02T05:11:05.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:11:05.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:11:05.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:11:05.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:11:05.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:11:05.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:11:05.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:11:05.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[24811]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 515 B/s rd, 0 op/s 2024-04-02T05:11:05.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:11:05.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:05.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:05.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-02T05:11:05.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-02T05:11:05.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-02T05:11:05.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-02T05:11:05.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-02T05:11:05.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-02T05:11:05.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:05.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:05.088 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:05.088 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:05.088 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:05.088 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:05.088 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:05.088 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:05.088 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:05.088 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:05.088 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:05.088 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:05.088 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:05.088 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:05.089 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:05.089 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:05.089 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Traceback (most recent call last): 2024-04-02T05:11:05.089 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T05:11:05.089 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T05:11:05.089 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T05:11:05.089 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T05:11:05.089 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T05:11:05.089 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: return self.event_loop.get_result(coro, timeout) 2024-04-02T05:11:05.090 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T05:11:05.090 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: return future.result(timeout) 2024-04-02T05:11:05.090 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T05:11:05.090 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: return self.__get_result() 2024-04-02T05:11:05.090 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T05:11:05.090 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: raise self._exception 2024-04-02T05:11:05.090 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T05:11:05.090 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: config_blobs=daemon_spec.final_config, 2024-04-02T05:11:05.090 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T05:11:05.090 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T05:11:05.090 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:11:05.090 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Creating ceph-iscsi config... 2024-04-02T05:11:05.090 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:11:05.091 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:11:05.091 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:11:05.091 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:05.091 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:05.091 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Traceback (most recent call last): 2024-04-02T05:11:05.091 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:11:05.091 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: return _run_code(code, main_globals, None, 2024-04-02T05:11:05.091 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:11:05.091 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: exec(code, run_globals) 2024-04-02T05:11:05.091 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:11:05.091 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:11:05.091 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:11:05.091 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:11:05.091 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:11:05.092 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:11:05.092 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:11:05.092 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:05.092 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:05.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:04 smithi192 ceph-mon[25697]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 515 B/s rd, 0 op/s 2024-04-02T05:11:05.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:04 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:11:05.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:04 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:05.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:04 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:05.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:04 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-02T05:11:05.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:04 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-02T05:11:05.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:04 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-02T05:11:05.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:04 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-02T05:11:05.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:04 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-02T05:11:05.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:04 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-02T05:11:05.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:04 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:05.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:04 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:05.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:04 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:05.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:04 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:05.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:04 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:05.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:04 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:05.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:04 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:05.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:04 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:05.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:04 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:05.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:04 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:05.179 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:04 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:05.180 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:04 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:05.180 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:04 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:05.180 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:04 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:05.545 INFO:teuthology.orchestra.run.smithi086.stdout:true 2024-04-02T05:11:06.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:05 smithi086 ceph-mon[24811]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:11:06.083 INFO:journalctl@ceph.mon.a.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T05:11:06.083 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:11:06.083 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:11:06.084 INFO:journalctl@ceph.mon.a.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:11:06.084 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:06.084 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:06.084 INFO:journalctl@ceph.mon.a.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:11:06.084 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:11:06.084 INFO:journalctl@ceph.mon.a.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:11:06.084 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:11:06.084 INFO:journalctl@ceph.mon.a.smithi086.stdout: exec(code, run_globals) 2024-04-02T05:11:06.084 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:11:06.084 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:11:06.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:11:06.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:11:06.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:11:06.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:11:06.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:11:06.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:06.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:06.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:11:06.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T05:11:06.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T05:11:06.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T05:11:06.086 INFO:journalctl@ceph.mon.a.smithi086.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T05:11:06.086 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T05:11:06.086 INFO:journalctl@ceph.mon.a.smithi086.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-02T05:11:06.086 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T05:11:06.086 INFO:journalctl@ceph.mon.a.smithi086.stdout: return future.result(timeout) 2024-04-02T05:11:06.086 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T05:11:06.086 INFO:journalctl@ceph.mon.a.smithi086.stdout: return self.__get_result() 2024-04-02T05:11:06.086 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T05:11:06.086 INFO:journalctl@ceph.mon.a.smithi086.stdout: raise self._exception 2024-04-02T05:11:06.086 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T05:11:06.086 INFO:journalctl@ceph.mon.a.smithi086.stdout: config_blobs=daemon_spec.final_config, 2024-04-02T05:11:06.087 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T05:11:06.087 INFO:journalctl@ceph.mon.a.smithi086.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T05:11:06.087 INFO:journalctl@ceph.mon.a.smithi086.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:11:06.087 INFO:journalctl@ceph.mon.a.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T05:11:06.087 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:11:06.087 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:11:06.087 INFO:journalctl@ceph.mon.a.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:11:06.087 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:06.087 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:06.087 INFO:journalctl@ceph.mon.a.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:11:06.087 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:11:06.087 INFO:journalctl@ceph.mon.a.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:11:06.088 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:11:06.088 INFO:journalctl@ceph.mon.a.smithi086.stdout: exec(code, run_globals) 2024-04-02T05:11:06.088 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:11:06.088 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:11:06.088 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:11:06.088 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:11:06.088 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:05 smithi086 ceph-mon[29505]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:11:06.088 INFO:journalctl@ceph.mon.c.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T05:11:06.089 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:11:06.089 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:11:06.089 INFO:journalctl@ceph.mon.c.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:11:06.089 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:06.089 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:06.089 INFO:journalctl@ceph.mon.c.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:11:06.089 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:11:06.089 INFO:journalctl@ceph.mon.c.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:11:06.089 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:11:06.089 INFO:journalctl@ceph.mon.c.smithi086.stdout: exec(code, run_globals) 2024-04-02T05:11:06.089 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:11:06.090 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:11:06.090 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:11:06.090 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:11:06.090 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:11:06.090 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:11:06.090 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:11:06.090 INFO:journalctl@ceph.mon.c.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:06.090 INFO:journalctl@ceph.mon.c.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:06.090 INFO:journalctl@ceph.mon.c.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:11:06.090 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T05:11:06.090 INFO:journalctl@ceph.mon.c.smithi086.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T05:11:06.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T05:11:06.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T05:11:06.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T05:11:06.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-02T05:11:06.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T05:11:06.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: return future.result(timeout) 2024-04-02T05:11:06.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T05:11:06.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: return self.__get_result() 2024-04-02T05:11:06.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T05:11:06.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: raise self._exception 2024-04-02T05:11:06.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T05:11:06.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: config_blobs=daemon_spec.final_config, 2024-04-02T05:11:06.092 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T05:11:06.092 INFO:journalctl@ceph.mon.c.smithi086.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T05:11:06.092 INFO:journalctl@ceph.mon.c.smithi086.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:11:06.092 INFO:journalctl@ceph.mon.c.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T05:11:06.092 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:11:06.092 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:11:06.092 INFO:journalctl@ceph.mon.c.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:11:06.092 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:06.092 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:06.092 INFO:journalctl@ceph.mon.c.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:11:06.092 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:11:06.092 INFO:journalctl@ceph.mon.c.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:11:06.093 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:11:06.093 INFO:journalctl@ceph.mon.c.smithi086.stdout: exec(code, run_globals) 2024-04-02T05:11:06.093 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:11:06.093 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:11:06.093 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:11:06.093 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:11:06.093 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:11:06.093 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:11:06.094 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:11:06.094 INFO:journalctl@ceph.mon.a.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:06.094 INFO:journalctl@ceph.mon.a.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:06.094 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:05 smithi086 ceph-mon[24811]: Upgrade: Setting container_image for all mgr 2024-04-02T05:11:06.094 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:05 smithi086 ceph-mon[24811]: Upgrade: Setting container_image for all crash 2024-04-02T05:11:06.094 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:05 smithi086 ceph-mon[24811]: Upgrade: Setting container_image for all mds 2024-04-02T05:11:06.094 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:05 smithi086 ceph-mon[24811]: Upgrade: Setting container_image for all rbd-mirror 2024-04-02T05:11:06.094 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:05 smithi086 ceph-mon[24811]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-02T05:11:06.094 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:05 smithi086 ceph-mon[24811]: Upgrade: Setting container_image for all ceph-exporter 2024-04-02T05:11:06.094 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:05 smithi086 ceph-mon[24811]: Upgrade: Setting container_image for all nfs 2024-04-02T05:11:06.094 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:05 smithi086 ceph-mon[24811]: Upgrade: Setting container_image for all nvmeof 2024-04-02T05:11:06.095 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:11:06.095 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:11:06.095 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:11:06.095 INFO:journalctl@ceph.mon.c.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:06.095 INFO:journalctl@ceph.mon.c.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:06.095 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:05 smithi086 ceph-mon[29505]: Upgrade: Setting container_image for all mgr 2024-04-02T05:11:06.095 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:05 smithi086 ceph-mon[29505]: Upgrade: Setting container_image for all crash 2024-04-02T05:11:06.095 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:05 smithi086 ceph-mon[29505]: Upgrade: Setting container_image for all mds 2024-04-02T05:11:06.095 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:05 smithi086 ceph-mon[29505]: Upgrade: Setting container_image for all rbd-mirror 2024-04-02T05:11:06.095 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:05 smithi086 ceph-mon[29505]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-02T05:11:06.095 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:05 smithi086 ceph-mon[29505]: Upgrade: Setting container_image for all ceph-exporter 2024-04-02T05:11:06.095 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:05 smithi086 ceph-mon[29505]: Upgrade: Setting container_image for all nfs 2024-04-02T05:11:06.096 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:05 smithi086 ceph-mon[29505]: Upgrade: Setting container_image for all nvmeof 2024-04-02T05:11:06.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:05 smithi192 ceph-mon[25697]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:11:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout: Creating ceph-iscsi config... 2024-04-02T05:11:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:11:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:11:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:11:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout: Traceback (most recent call last): 2024-04-02T05:11:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:11:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:11:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:11:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout: exec(code, run_globals) 2024-04-02T05:11:06.178 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:11:06.179 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:11:06.179 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:11:06.179 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:11:06.179 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:11:06.179 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:11:06.179 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:11:06.179 INFO:journalctl@ceph.mon.b.smithi192.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:06.179 INFO:journalctl@ceph.mon.b.smithi192.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:06.179 INFO:journalctl@ceph.mon.b.smithi192.stdout: Traceback (most recent call last): 2024-04-02T05:11:06.179 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T05:11:06.179 INFO:journalctl@ceph.mon.b.smithi192.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T05:11:06.179 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T05:11:06.179 INFO:journalctl@ceph.mon.b.smithi192.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T05:11:06.179 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T05:11:06.180 INFO:journalctl@ceph.mon.b.smithi192.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-02T05:11:06.180 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T05:11:06.180 INFO:journalctl@ceph.mon.b.smithi192.stdout: return future.result(timeout) 2024-04-02T05:11:06.180 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T05:11:06.180 INFO:journalctl@ceph.mon.b.smithi192.stdout: return self.__get_result() 2024-04-02T05:11:06.180 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T05:11:06.180 INFO:journalctl@ceph.mon.b.smithi192.stdout: raise self._exception 2024-04-02T05:11:06.180 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T05:11:06.180 INFO:journalctl@ceph.mon.b.smithi192.stdout: config_blobs=daemon_spec.final_config, 2024-04-02T05:11:06.180 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T05:11:06.180 INFO:journalctl@ceph.mon.b.smithi192.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T05:11:06.180 INFO:journalctl@ceph.mon.b.smithi192.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:11:06.180 INFO:journalctl@ceph.mon.b.smithi192.stdout: Creating ceph-iscsi config... 2024-04-02T05:11:06.180 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:11:06.181 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:11:06.181 INFO:journalctl@ceph.mon.b.smithi192.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:11:06.181 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:06.181 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:06.181 INFO:journalctl@ceph.mon.b.smithi192.stdout: Traceback (most recent call last): 2024-04-02T05:11:06.181 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:11:06.181 INFO:journalctl@ceph.mon.b.smithi192.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:11:06.181 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:11:06.181 INFO:journalctl@ceph.mon.b.smithi192.stdout: exec(code, run_globals) 2024-04-02T05:11:06.181 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:11:06.181 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:11:06.181 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:11:06.181 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:11:06.182 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:11:06.182 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:11:06.182 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:11:06.182 INFO:journalctl@ceph.mon.b.smithi192.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:06.182 INFO:journalctl@ceph.mon.b.smithi192.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:06.182 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:05 smithi192 ceph-mon[25697]: Upgrade: Setting container_image for all mgr 2024-04-02T05:11:06.182 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:05 smithi192 ceph-mon[25697]: Upgrade: Setting container_image for all crash 2024-04-02T05:11:06.182 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:05 smithi192 ceph-mon[25697]: Upgrade: Setting container_image for all mds 2024-04-02T05:11:06.182 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:05 smithi192 ceph-mon[25697]: Upgrade: Setting container_image for all rbd-mirror 2024-04-02T05:11:06.182 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:05 smithi192 ceph-mon[25697]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-02T05:11:06.182 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:05 smithi192 ceph-mon[25697]: Upgrade: Setting container_image for all ceph-exporter 2024-04-02T05:11:06.183 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:05 smithi192 ceph-mon[25697]: Upgrade: Setting container_image for all nfs 2024-04-02T05:11:06.183 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:05 smithi192 ceph-mon[25697]: Upgrade: Setting container_image for all nvmeof 2024-04-02T05:11:06.203 INFO:teuthology.orchestra.run.smithi086.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T05:11:06.203 INFO:teuthology.orchestra.run.smithi086.stdout:alertmanager.a smithi086 *:9093,9094 running (13m) 28s ago 21m 22.7M - 0.23.0 ba2b418f427c b2769c8e287a 2024-04-02T05:11:06.204 INFO:teuthology.orchestra.run.smithi086.stdout:grafana.a smithi192 *:3000 error 9s ago 20m - - 2024-04-02T05:11:06.204 INFO:teuthology.orchestra.run.smithi086.stdout:iscsi.foo.smithi086.rsbqwx smithi086 running (3m) 28s ago 20m 61.7M - 3.5 e1d6a67b021e 2b9643017ea8 2024-04-02T05:11:06.204 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.x smithi192 *:8443,9283,8765 running (13s) 9s ago 25m 64.6M - 19.0.0-1979-g6cec793d 78f830190839 dd1e6d0f0a95 2024-04-02T05:11:06.204 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.y smithi086 *:8443,9283,8765 running (9m) 28s ago 27m 474M - 19.0.0-1979-g6cec793d 78f830190839 3dca54a018ab 2024-04-02T05:11:06.204 INFO:teuthology.orchestra.run.smithi086.stdout:mon.a smithi086 running (27m) 28s ago 27m 75.8M 2048M 17.2.0 e1d6a67b021e b6c507ab96f8 2024-04-02T05:11:06.204 INFO:teuthology.orchestra.run.smithi086.stdout:mon.b smithi192 running (25m) 9s ago 25m 59.1M 2048M 17.2.0 e1d6a67b021e 4dadd8d1c318 2024-04-02T05:11:06.204 INFO:teuthology.orchestra.run.smithi086.stdout:mon.c smithi086 running (25m) 28s ago 25m 52.6M 2048M 17.2.0 e1d6a67b021e 49275483a2d4 2024-04-02T05:11:06.204 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.a smithi086 *:9100 running (13m) 28s ago 21m 17.5M - 1.3.1 1dbe0e931976 843a5e9887fb 2024-04-02T05:11:06.204 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.b smithi192 *:9100 running (12m) 9s ago 21m 17.1M - 1.3.1 1dbe0e931976 bd849d6c4fb0 2024-04-02T05:11:06.205 INFO:teuthology.orchestra.run.smithi086.stdout:osd.0 smithi086 running (25m) 28s ago 25m 56.8M 2992M 17.2.0 e1d6a67b021e 6d90c60d1c71 2024-04-02T05:11:06.205 INFO:teuthology.orchestra.run.smithi086.stdout:osd.1 smithi086 running (24m) 28s ago 24m 55.2M 2992M 17.2.0 e1d6a67b021e 203689f71603 2024-04-02T05:11:06.205 INFO:teuthology.orchestra.run.smithi086.stdout:osd.2 smithi086 running (24m) 28s ago 24m 49.7M 2992M 17.2.0 e1d6a67b021e 472c860a9af8 2024-04-02T05:11:06.205 INFO:teuthology.orchestra.run.smithi086.stdout:osd.3 smithi086 running (24m) 28s ago 24m 52.5M 2992M 17.2.0 e1d6a67b021e 327538dac44e 2024-04-02T05:11:06.205 INFO:teuthology.orchestra.run.smithi086.stdout:osd.4 smithi192 running (23m) 9s ago 23m 55.9M 3248M 17.2.0 e1d6a67b021e 6535d7919a2d 2024-04-02T05:11:06.205 INFO:teuthology.orchestra.run.smithi086.stdout:osd.5 smithi192 running (23m) 9s ago 23m 54.7M 3248M 17.2.0 e1d6a67b021e 7259249605f2 2024-04-02T05:11:06.205 INFO:teuthology.orchestra.run.smithi086.stdout:osd.6 smithi192 running (22m) 9s ago 22m 52.6M 3248M 17.2.0 e1d6a67b021e bb7d171b02a1 2024-04-02T05:11:06.205 INFO:teuthology.orchestra.run.smithi086.stdout:osd.7 smithi192 running (22m) 9s ago 22m 54.5M 3248M 17.2.0 e1d6a67b021e 95f5a0353999 2024-04-02T05:11:06.205 INFO:teuthology.orchestra.run.smithi086.stdout:prometheus.a smithi192 *:9095 running (19s) 9s ago 21m 53.1M - 2.33.4 514e6a882f6e f61f481e7475 2024-04-02T05:11:06.205 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi086.bxzxlp smithi086 *:8000 running (20m) 28s ago 20m 77.4M - 17.2.0 e1d6a67b021e 6725785fbc53 2024-04-02T05:11:06.205 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi192.pmlbot smithi192 *:8000 running (20m) 9s ago 20m 76.9M - 17.2.0 e1d6a67b021e 5ae3638f8b1c 2024-04-02T05:11:06.608 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:11:06.608 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": { 2024-04-02T05:11:06.608 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-02T05:11:06.608 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:11:06.608 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": { 2024-04-02T05:11:06.608 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 2 2024-04-02T05:11:06.608 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:11:06.608 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": { 2024-04-02T05:11:06.608 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-02T05:11:06.608 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:11:06.608 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": {}, 2024-04-02T05:11:06.609 INFO:teuthology.orchestra.run.smithi086.stdout: "rgw": { 2024-04-02T05:11:06.609 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-02T05:11:06.609 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:11:06.609 INFO:teuthology.orchestra.run.smithi086.stdout: "overall": { 2024-04-02T05:11:06.609 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-02T05:11:06.609 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 2 2024-04-02T05:11:06.609 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-04-02T05:11:06.609 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:11:06.929 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:06 smithi086 ceph-mon[24811]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:11:06.929 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:06 smithi086 ceph-mon[24811]: from='client.25429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:11:06.929 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:06 smithi086 ceph-mon[24811]: from='client.25435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:11:06.929 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:06 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/2417672009' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:06.966 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:11:06.967 INFO:teuthology.orchestra.run.smithi086.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", 2024-04-02T05:11:06.967 INFO:teuthology.orchestra.run.smithi086.stdout: "in_progress": true, 2024-04-02T05:11:06.967 INFO:teuthology.orchestra.run.smithi086.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-04-02T05:11:06.967 INFO:teuthology.orchestra.run.smithi086.stdout: "services_complete": [ 2024-04-02T05:11:06.967 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr" 2024-04-02T05:11:06.967 INFO:teuthology.orchestra.run.smithi086.stdout: ], 2024-04-02T05:11:06.967 INFO:teuthology.orchestra.run.smithi086.stdout: "progress": "2/2 daemons upgraded", 2024-04-02T05:11:06.967 INFO:teuthology.orchestra.run.smithi086.stdout: "message": "Currently upgrading node-exporter daemons", 2024-04-02T05:11:06.967 INFO:teuthology.orchestra.run.smithi086.stdout: "is_paused": false 2024-04-02T05:11:06.967 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:11:07.254 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:06 smithi086 ceph-mon[29505]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:11:07.254 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:06 smithi086 ceph-mon[29505]: from='client.25429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:11:07.254 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:06 smithi086 ceph-mon[29505]: from='client.25435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:11:07.254 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:06 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/2417672009' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:07.255 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:07 smithi086 systemd[1]: Stopping Ceph node-exporter.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:11:07.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:06 smithi192 ceph-mon[25697]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:11:07.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:06 smithi192 ceph-mon[25697]: from='client.25429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:11:07.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:06 smithi192 ceph-mon[25697]: from='client.25435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:11:07.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:06 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/2417672009' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:07.553 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:07 smithi086 podman[77179]: 2024-04-02 05:11:07.25159268 +0000 UTC m=+0.107702133 container died 843a5e9887fba59327154514ed1f112b3c2406c7bf7efe073a54954d144d7f2e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-02T05:11:07.833 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:07 smithi086 podman[77179]: 2024-04-02 05:11:07.640965106 +0000 UTC m=+0.497074557 container cleanup 843a5e9887fba59327154514ed1f112b3c2406c7bf7efe073a54954d144d7f2e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-02T05:11:07.833 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:07 smithi086 bash[77179]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a 2024-04-02T05:11:08.215 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:07 smithi086 ceph-mon[29505]: from='client.15552 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:11:08.215 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:07 smithi086 ceph-mon[29505]: Upgrade: Updating node-exporter.a (1/2) 2024-04-02T05:11:08.215 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:07 smithi086 ceph-mon[29505]: Deploying daemon node-exporter.a on smithi086 2024-04-02T05:11:08.215 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:07 smithi086 ceph-mon[29505]: from='client.15564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:11:08.216 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:07 smithi086 ceph-mon[24811]: from='client.15552 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:11:08.216 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:07 smithi086 ceph-mon[24811]: Upgrade: Updating node-exporter.a (1/2) 2024-04-02T05:11:08.216 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:07 smithi086 ceph-mon[24811]: Deploying daemon node-exporter.a on smithi086 2024-04-02T05:11:08.216 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:07 smithi086 ceph-mon[24811]: from='client.15564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:11:08.216 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:07 smithi086 podman[77193]: 2024-04-02 05:11:07.917306128 +0000 UTC m=+0.669693791 container remove 843a5e9887fba59327154514ed1f112b3c2406c7bf7efe073a54954d144d7f2e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-02T05:11:08.216 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:07 smithi086 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-02T05:11:08.216 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:08 smithi086 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-02T05:11:08.216 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:08 smithi086 systemd[1]: Stopped Ceph node-exporter.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:11:08.217 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:08 smithi086 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@node-exporter.a.service: Consumed 5.154s CPU time. 2024-04-02T05:11:08.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:07 smithi192 ceph-mon[25697]: from='client.15552 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:11:08.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:07 smithi192 ceph-mon[25697]: Upgrade: Updating node-exporter.a (1/2) 2024-04-02T05:11:08.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:07 smithi192 ceph-mon[25697]: Deploying daemon node-exporter.a on smithi086 2024-04-02T05:11:08.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:07 smithi192 ceph-mon[25697]: from='client.15564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:11:08.583 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:08 smithi086 systemd[1]: Starting Ceph node-exporter.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:11:08.583 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:08 smithi086 bash[77323]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-04-02T05:11:09.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:08 smithi086 ceph-mon[29505]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:11:09.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:08 smithi086 ceph-mon[24811]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:11:09.333 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:08 smithi086 bash[77323]: Getting image source signatures 2024-04-02T05:11:09.333 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:08 smithi086 bash[77323]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-04-02T05:11:09.334 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:08 smithi086 bash[77323]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-04-02T05:11:09.334 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:08 smithi086 bash[77323]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-04-02T05:11:09.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:08 smithi192 ceph-mon[25697]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:11:10.834 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:10 smithi086 bash[77323]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-04-02T05:11:10.834 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:10 smithi086 bash[77323]: Writing manifest to image destination 2024-04-02T05:11:11.247 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:10 smithi086 ceph-mon[24811]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 619 B/s rd, 0 op/s 2024-04-02T05:11:11.247 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:10 smithi086 ceph-mon[29505]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 619 B/s rd, 0 op/s 2024-04-02T05:11:11.247 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:10 smithi086 podman[77323]: 2024-04-02 05:11:10.911052842 +0000 UTC m=+2.601231766 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-02T05:11:11.248 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 podman[77323]: 2024-04-02 05:11:11.034605268 +0000 UTC m=+2.724784222 container create 05a844569d7b30fcacd71836eb9b14c5b7278d60f158c60f9ed8f4e80c627c0b (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-02T05:11:11.248 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 podman[77323]: 2024-04-02 05:11:11.245174147 +0000 UTC m=+2.935353071 container init 05a844569d7b30fcacd71836eb9b14c5b7278d60f158c60f9ed8f4e80c627c0b (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-02T05:11:11.380 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:10 smithi192 ceph-mon[25697]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 619 B/s rd, 0 op/s 2024-04-02T05:11:11.583 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 podman[77323]: 2024-04-02 05:11:11.248182491 +0000 UTC m=+2.938361400 container start 05a844569d7b30fcacd71836eb9b14c5b7278d60f158c60f9ed8f4e80c627c0b (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-02T05:11:11.583 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.252Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-02T05:11:11.583 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.252Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-02T05:11:11.583 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.253Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-02T05:11:11.583 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.253Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-02T05:11:11.584 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.253Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-02T05:11:11.584 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.253Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-02T05:11:11.584 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-02T05:11:11.584 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=arp 2024-04-02T05:11:11.584 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-02T05:11:11.584 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-02T05:11:11.584 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-02T05:11:11.584 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-02T05:11:11.584 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-02T05:11:11.584 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-02T05:11:11.584 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-02T05:11:11.584 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-02T05:11:11.585 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=edac 2024-04-02T05:11:11.585 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-02T05:11:11.585 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-02T05:11:11.585 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-02T05:11:11.585 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-02T05:11:11.585 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-02T05:11:11.585 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-02T05:11:11.585 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-02T05:11:11.585 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-02T05:11:11.585 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-02T05:11:11.586 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-02T05:11:11.586 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-02T05:11:11.586 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-02T05:11:11.586 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-02T05:11:11.586 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-02T05:11:11.586 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-02T05:11:11.586 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-02T05:11:11.586 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=os 2024-04-02T05:11:11.586 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-02T05:11:11.586 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-02T05:11:11.586 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-02T05:11:11.586 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-02T05:11:11.586 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-02T05:11:11.587 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-02T05:11:11.587 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-02T05:11:11.587 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=stat 2024-04-02T05:11:11.587 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-02T05:11:11.587 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-02T05:11:11.587 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-02T05:11:11.587 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=time 2024-04-02T05:11:11.587 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-02T05:11:11.587 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=uname 2024-04-02T05:11:11.587 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-02T05:11:11.587 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-02T05:11:11.587 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-02T05:11:11.587 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-02T05:11:11.587 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a[77374]: ts=2024-04-02T05:11:11.254Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-02T05:11:11.588 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 bash[77323]: 05a844569d7b30fcacd71836eb9b14c5b7278d60f158c60f9ed8f4e80c627c0b 2024-04-02T05:11:11.588 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:11:11 smithi086 systemd[1]: Started Ceph node-exporter.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:11:12.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:12 smithi192 ceph-mon[25697]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 614 B/s rd, 0 op/s 2024-04-02T05:11:12.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:12 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:12.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:12 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:12.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:12 smithi086 ceph-mon[29505]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 614 B/s rd, 0 op/s 2024-04-02T05:11:12.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:12 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:12.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:12 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:12.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:12 smithi086 ceph-mon[24811]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 614 B/s rd, 0 op/s 2024-04-02T05:11:12.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:12 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:12.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:12 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:13.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:13 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:11:13] "GET /metrics HTTP/1.1" 200 37986 "" "Prometheus/2.33.4" 2024-04-02T05:11:13.927 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:13 smithi192 systemd[1]: Stopping Ceph node-exporter.b for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:11:14.301 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:13 smithi192 podman[62722]: 2024-04-02 05:11:13.991696316 +0000 UTC m=+0.137783334 container died bd849d6c4fb0ce13ce45e53b9760a2e937f86e769246ed1b848aa4bf90eae0aa (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-02T05:11:14.601 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:14 smithi192 podman[62722]: 2024-04-02 05:11:14.397869606 +0000 UTC m=+0.543956629 container cleanup bd849d6c4fb0ce13ce45e53b9760a2e937f86e769246ed1b848aa4bf90eae0aa (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-02T05:11:14.601 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:14 smithi192 bash[62722]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b 2024-04-02T05:11:14.601 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:14 smithi192 ceph-mon[25697]: Upgrade: Updating node-exporter.b (2/2) 2024-04-02T05:11:14.601 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:14 smithi192 ceph-mon[25697]: Deploying daemon node-exporter.b on smithi192 2024-04-02T05:11:14.602 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:14 smithi192 ceph-mon[25697]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-02T05:11:14.823 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:14 smithi086 ceph-mon[29505]: Upgrade: Updating node-exporter.b (2/2) 2024-04-02T05:11:14.823 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:14 smithi086 ceph-mon[29505]: Deploying daemon node-exporter.b on smithi192 2024-04-02T05:11:14.823 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:14 smithi086 ceph-mon[29505]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-02T05:11:14.823 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:14 smithi086 ceph-mon[24811]: Upgrade: Updating node-exporter.b (2/2) 2024-04-02T05:11:14.823 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:14 smithi086 ceph-mon[24811]: Deploying daemon node-exporter.b on smithi192 2024-04-02T05:11:14.824 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:14 smithi086 ceph-mon[24811]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-02T05:11:14.927 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:14 smithi192 podman[62737]: 2024-04-02 05:11:14.599211343 +0000 UTC m=+0.608245800 container remove bd849d6c4fb0ce13ce45e53b9760a2e937f86e769246ed1b848aa4bf90eae0aa (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-02T05:11:14.927 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:14 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-02T05:11:14.927 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:14 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-02T05:11:14.928 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:14 smithi192 systemd[1]: Stopped Ceph node-exporter.b for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:11:14.928 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:14 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@node-exporter.b.service: Consumed 5.339s CPU time. 2024-04-02T05:11:15.427 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:15 smithi192 systemd[1]: Starting Ceph node-exporter.b for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:11:15.427 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:15 smithi192 bash[62868]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-04-02T05:11:15.927 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:15 smithi192 bash[62868]: Getting image source signatures 2024-04-02T05:11:15.927 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:15 smithi192 bash[62868]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-04-02T05:11:15.928 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:15 smithi192 bash[62868]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-04-02T05:11:15.928 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:15 smithi192 bash[62868]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-04-02T05:11:17.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:16 smithi086 ceph-mon[24811]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-02T05:11:17.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:16 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/485791666' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T05:11:17.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:16 smithi086 ceph-mon[29505]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-02T05:11:17.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:16 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/485791666' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T05:11:17.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:16 smithi192 ceph-mon[25697]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-02T05:11:17.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:16 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/485791666' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T05:11:17.927 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:17 smithi192 bash[62868]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-04-02T05:11:17.927 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:17 smithi192 bash[62868]: Writing manifest to image destination 2024-04-02T05:11:18.199 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:17 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:11:18.200 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 podman[62868]: 2024-04-02 05:11:17.964063219 +0000 UTC m=+2.863583501 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-02T05:11:18.200 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 podman[62868]: 2024-04-02 05:11:18.079484734 +0000 UTC m=+2.979005044 container create 1a7d6788d58bdb0973997a9b7fde482a8eee8f4ac65cdf6fbe64b1851ff30451 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-02T05:11:18.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:17 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:11:18.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:17 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:11:18.610 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 podman[62868]: 2024-04-02 05:11:18.298527451 +0000 UTC m=+3.198047745 container init 1a7d6788d58bdb0973997a9b7fde482a8eee8f4ac65cdf6fbe64b1851ff30451 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-02T05:11:18.611 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 podman[62868]: 2024-04-02 05:11:18.30170443 +0000 UTC m=+3.201224695 container start 1a7d6788d58bdb0973997a9b7fde482a8eee8f4ac65cdf6fbe64b1851ff30451 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-02T05:11:18.611 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.305Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-02T05:11:18.611 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.305Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-02T05:11:18.611 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.306Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-02T05:11:18.611 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.306Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-02T05:11:18.611 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.306Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-02T05:11:18.611 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.306Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-02T05:11:18.611 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-02T05:11:18.611 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=arp 2024-04-02T05:11:18.611 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-02T05:11:18.612 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-02T05:11:18.612 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-02T05:11:18.612 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-02T05:11:18.612 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-02T05:11:18.612 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-02T05:11:18.612 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-02T05:11:18.613 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-02T05:11:18.613 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=edac 2024-04-02T05:11:18.613 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-02T05:11:18.613 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-02T05:11:18.613 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-02T05:11:18.613 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-02T05:11:18.613 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-02T05:11:18.613 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-02T05:11:18.613 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-02T05:11:18.613 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-02T05:11:18.613 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-02T05:11:18.614 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-02T05:11:18.614 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-02T05:11:18.614 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-02T05:11:18.614 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-02T05:11:18.614 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-02T05:11:18.614 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-02T05:11:18.614 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-02T05:11:18.614 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=os 2024-04-02T05:11:18.614 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-02T05:11:18.614 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-02T05:11:18.614 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-02T05:11:18.615 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-02T05:11:18.615 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-02T05:11:18.615 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-02T05:11:18.615 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-02T05:11:18.615 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=stat 2024-04-02T05:11:18.615 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-02T05:11:18.615 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-02T05:11:18.615 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-02T05:11:18.615 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=time 2024-04-02T05:11:18.615 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-02T05:11:18.615 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=uname 2024-04-02T05:11:18.615 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-02T05:11:18.615 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-02T05:11:18.616 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-02T05:11:18.616 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-02T05:11:18.616 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b[62920]: ts=2024-04-02T05:11:18.307Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-02T05:11:18.617 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 bash[62868]: 1a7d6788d58bdb0973997a9b7fde482a8eee8f4ac65cdf6fbe64b1851ff30451 2024-04-02T05:11:18.617 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:11:18 smithi192 systemd[1]: Started Ceph node-exporter.b for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:11:19.284 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:18 smithi086 ceph-mon[29505]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:11:19.284 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:18 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:19.284 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:18 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:19.284 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:18 smithi086 ceph-mon[24811]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:11:19.284 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:18 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:19.285 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:18 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:19.289 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-mon[25697]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:11:19.289 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:19.289 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:18 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:21.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:20 smithi192 ceph-mon[25697]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-02T05:11:21.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:20 smithi086 ceph-mon[24811]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-02T05:11:21.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:20 smithi086 ceph-mon[29505]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-02T05:11:23.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:22 smithi086 ceph-mon[24811]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-02T05:11:23.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:23.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:23.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:23.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:23.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:22 smithi086 ceph-mon[29505]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-02T05:11:23.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:23.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:23.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:23.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:23.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:22 smithi192 ceph-mon[25697]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-02T05:11:23.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:22 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:23.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:22 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:23.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:22 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:23.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:22 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:23.714 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:11:23] "GET /metrics HTTP/1.1" 200 37988 "" "Prometheus/2.33.4" 2024-04-02T05:11:24.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:24 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:24.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:24 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:24.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:24 smithi192 ceph-mon[25697]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-02T05:11:24.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:24 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:24.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:24 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:24.574 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:24 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:24.574 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:24 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:24.574 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:24 smithi086 ceph-mon[29505]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-02T05:11:24.574 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:24 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:24.574 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:24 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:24.575 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:24 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:24.575 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:24 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:24.575 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:24 smithi086 ceph-mon[24811]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-02T05:11:24.575 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:24 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:24.575 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:24 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:27.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:26 smithi086 ceph-mon[24811]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-02T05:11:27.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:26 smithi086 ceph-mon[24811]: from='client.25471 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:11:27.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:26 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:27.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:26 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:27.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:26 smithi086 ceph-mon[29505]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-02T05:11:27.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:26 smithi086 ceph-mon[29505]: from='client.25471 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:11:27.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:26 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:27.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:26 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:27.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:26 smithi192 ceph-mon[25697]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2024-04-02T05:11:27.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:26 smithi192 ceph-mon[25697]: from='client.25471 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:11:27.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:26 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:27.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:26 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:28.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:28 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:28.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:28 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:28.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:28 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:11:28.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:28 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:11:28.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:28 smithi086 ceph-mon[24811]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 687 B/s rd, 0 op/s 2024-04-02T05:11:28.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:28 smithi086 ceph-mon[24811]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 825 B/s rd, 0 op/s 2024-04-02T05:11:28.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:28 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:28.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:28 smithi086 ceph-mon[24811]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T05:11:28.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:28 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:11:28.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:28 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:11:28.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:28 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:11:28.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:28 smithi086 ceph-mon[24811]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T05:11:28.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:28 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:28.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:28 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:28.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:28 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:11:28.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:28 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:11:28.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:28 smithi086 ceph-mon[29505]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 687 B/s rd, 0 op/s 2024-04-02T05:11:28.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:28 smithi086 ceph-mon[29505]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 825 B/s rd, 0 op/s 2024-04-02T05:11:28.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:28 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:28.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:28 smithi086 ceph-mon[29505]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T05:11:28.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:28 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:11:28.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:28 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:11:28.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:28 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:11:28.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:28 smithi086 ceph-mon[29505]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T05:11:28.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:28 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:28.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:28 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:28.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:28 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:11:28.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:28 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:11:28.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:28 smithi192 ceph-mon[25697]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 687 B/s rd, 0 op/s 2024-04-02T05:11:28.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:28 smithi192 ceph-mon[25697]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 825 B/s rd, 0 op/s 2024-04-02T05:11:28.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:28 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:28.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:28 smithi192 ceph-mon[25697]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T05:11:28.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:28 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:11:28.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:28 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:11:28.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:28 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:11:28.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:28 smithi192 ceph-mon[25697]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T05:11:30.616 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:30 smithi192 ceph-mon[25697]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T05:11:30.692 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-mon[29505]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T05:11:30.693 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-mon[24811]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T05:11:31.084 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:11:30.757+0000 7f180812e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:11:31.084 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Creating ceph-iscsi config... 2024-04-02T05:11:31.084 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:11:31.084 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:11:31.084 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:11:31.084 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:31.084 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:31.084 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Traceback (most recent call last): 2024-04-02T05:11:31.084 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:11:31.084 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: return _run_code(code, main_globals, None, 2024-04-02T05:11:31.085 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:11:31.085 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: exec(code, run_globals) 2024-04-02T05:11:31.085 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:11:31.085 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:11:31.085 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:11:31.085 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:11:31.085 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:11:31.085 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:11:31.085 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:11:31.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:31.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:31.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Traceback (most recent call last): 2024-04-02T05:11:31.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T05:11:31.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T05:11:31.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T05:11:31.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T05:11:31.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T05:11:31.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: return self.event_loop.get_result(coro, timeout) 2024-04-02T05:11:31.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T05:11:31.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: return future.result(timeout) 2024-04-02T05:11:31.087 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T05:11:31.087 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: return self.__get_result() 2024-04-02T05:11:31.087 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T05:11:31.087 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: raise self._exception 2024-04-02T05:11:31.087 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T05:11:31.087 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: config_blobs=daemon_spec.final_config, 2024-04-02T05:11:31.087 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T05:11:31.087 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T05:11:31.087 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:11:31.087 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Creating ceph-iscsi config... 2024-04-02T05:11:31.087 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:11:31.087 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:11:31.087 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:11:31.088 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:31.088 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:31.088 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Traceback (most recent call last): 2024-04-02T05:11:31.088 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:11:31.088 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: return _run_code(code, main_globals, None, 2024-04-02T05:11:31.088 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:11:31.088 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: exec(code, run_globals) 2024-04-02T05:11:31.088 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:11:31.088 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:11:31.088 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:11:31.088 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:11:31.088 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:11:31.088 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:11:31.088 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:11:31.089 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:31.089 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:31.640 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:31 smithi192 ceph-mon[25697]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:11:31.640 INFO:journalctl@ceph.mon.b.smithi192.stdout: Creating ceph-iscsi config... 2024-04-02T05:11:31.640 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:11:31.640 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:11:31.640 INFO:journalctl@ceph.mon.b.smithi192.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:11:31.640 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:31.641 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:31.641 INFO:journalctl@ceph.mon.b.smithi192.stdout: Traceback (most recent call last): 2024-04-02T05:11:31.641 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:11:31.641 INFO:journalctl@ceph.mon.b.smithi192.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:11:31.641 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:11:31.641 INFO:journalctl@ceph.mon.b.smithi192.stdout: exec(code, run_globals) 2024-04-02T05:11:31.641 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:11:31.641 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:11:31.641 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:11:31.641 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:11:31.642 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:11:31.642 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:11:31.642 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:11:31.642 INFO:journalctl@ceph.mon.b.smithi192.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:31.642 INFO:journalctl@ceph.mon.b.smithi192.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:31.642 INFO:journalctl@ceph.mon.b.smithi192.stdout: Traceback (most recent call last): 2024-04-02T05:11:31.642 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T05:11:31.642 INFO:journalctl@ceph.mon.b.smithi192.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T05:11:31.642 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T05:11:31.642 INFO:journalctl@ceph.mon.b.smithi192.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T05:11:31.643 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T05:11:31.643 INFO:journalctl@ceph.mon.b.smithi192.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-02T05:11:31.643 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T05:11:31.643 INFO:journalctl@ceph.mon.b.smithi192.stdout: return future.result(timeout) 2024-04-02T05:11:31.643 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T05:11:31.643 INFO:journalctl@ceph.mon.b.smithi192.stdout: return self.__get_result() 2024-04-02T05:11:31.643 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T05:11:31.643 INFO:journalctl@ceph.mon.b.smithi192.stdout: raise self._exception 2024-04-02T05:11:31.643 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T05:11:31.643 INFO:journalctl@ceph.mon.b.smithi192.stdout: config_blobs=daemon_spec.final_config, 2024-04-02T05:11:31.643 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T05:11:31.643 INFO:journalctl@ceph.mon.b.smithi192.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T05:11:31.644 INFO:journalctl@ceph.mon.b.smithi192.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:11:31.644 INFO:journalctl@ceph.mon.b.smithi192.stdout: Creating ceph-iscsi config... 2024-04-02T05:11:31.644 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:11:31.644 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:11:31.644 INFO:journalctl@ceph.mon.b.smithi192.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:11:31.644 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:31.644 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:31.644 INFO:journalctl@ceph.mon.b.smithi192.stdout: Traceback (most recent call last): 2024-04-02T05:11:31.644 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:11:31.644 INFO:journalctl@ceph.mon.b.smithi192.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:11:31.644 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:11:31.644 INFO:journalctl@ceph.mon.b.smithi192.stdout: exec(code, run_globals) 2024-04-02T05:11:31.644 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:11:31.644 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:11:31.644 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:11:31.645 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:11:31.645 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:11:31.645 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:11:31.645 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:11:31.645 INFO:journalctl@ceph.mon.b.smithi192.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:31.645 INFO:journalctl@ceph.mon.b.smithi192.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:31.645 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:31 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:11:31.645 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:31 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:31.645 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:31 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:31.645 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:31 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:31.645 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:31 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:31.645 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:31 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:31.645 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:31 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:31.645 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:31 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:31.646 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:31 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:31.646 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:31 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:31.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:31 smithi086 ceph-mon[24811]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:11:31.833 INFO:journalctl@ceph.mon.a.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T05:11:31.833 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:11:31.833 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:11:31.833 INFO:journalctl@ceph.mon.a.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:11:31.834 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:31.834 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:31.834 INFO:journalctl@ceph.mon.a.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:11:31.834 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:11:31.834 INFO:journalctl@ceph.mon.a.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:11:31.834 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:11:31.834 INFO:journalctl@ceph.mon.a.smithi086.stdout: exec(code, run_globals) 2024-04-02T05:11:31.834 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:11:31.835 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:11:31.835 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:11:31.835 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:11:31.835 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:11:31.835 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:11:31.835 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:11:31.835 INFO:journalctl@ceph.mon.a.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:31.835 INFO:journalctl@ceph.mon.a.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:31.835 INFO:journalctl@ceph.mon.a.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:11:31.835 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T05:11:31.836 INFO:journalctl@ceph.mon.a.smithi086.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T05:11:31.836 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T05:11:31.836 INFO:journalctl@ceph.mon.a.smithi086.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T05:11:31.836 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T05:11:31.836 INFO:journalctl@ceph.mon.a.smithi086.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-02T05:11:31.836 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T05:11:31.836 INFO:journalctl@ceph.mon.a.smithi086.stdout: return future.result(timeout) 2024-04-02T05:11:31.836 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T05:11:31.836 INFO:journalctl@ceph.mon.a.smithi086.stdout: return self.__get_result() 2024-04-02T05:11:31.836 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T05:11:31.836 INFO:journalctl@ceph.mon.a.smithi086.stdout: raise self._exception 2024-04-02T05:11:31.836 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T05:11:31.836 INFO:journalctl@ceph.mon.a.smithi086.stdout: config_blobs=daemon_spec.final_config, 2024-04-02T05:11:31.837 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T05:11:31.837 INFO:journalctl@ceph.mon.a.smithi086.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T05:11:31.837 INFO:journalctl@ceph.mon.a.smithi086.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:11:31.837 INFO:journalctl@ceph.mon.a.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T05:11:31.837 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:11:31.837 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:11:31.837 INFO:journalctl@ceph.mon.a.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:11:31.837 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:31.837 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:31.837 INFO:journalctl@ceph.mon.a.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:11:31.837 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:11:31.837 INFO:journalctl@ceph.mon.a.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:11:31.837 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:11:31.838 INFO:journalctl@ceph.mon.a.smithi086.stdout: exec(code, run_globals) 2024-04-02T05:11:31.838 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:11:31.838 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:11:31.838 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:11:31.838 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:11:31.838 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:31 smithi086 ceph-mon[29505]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:11:31.838 INFO:journalctl@ceph.mon.c.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T05:11:31.838 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:11:31.839 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:11:31.839 INFO:journalctl@ceph.mon.c.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:11:31.839 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:31.839 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:31.839 INFO:journalctl@ceph.mon.c.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:11:31.839 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:11:31.839 INFO:journalctl@ceph.mon.c.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:11:31.839 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:11:31.839 INFO:journalctl@ceph.mon.c.smithi086.stdout: exec(code, run_globals) 2024-04-02T05:11:31.839 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:11:31.840 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:11:31.840 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:11:31.840 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:11:31.840 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:11:31.840 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:11:31.840 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:11:31.840 INFO:journalctl@ceph.mon.c.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:31.840 INFO:journalctl@ceph.mon.c.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:31.840 INFO:journalctl@ceph.mon.c.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:11:31.841 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T05:11:31.841 INFO:journalctl@ceph.mon.c.smithi086.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T05:11:31.841 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T05:11:31.841 INFO:journalctl@ceph.mon.c.smithi086.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T05:11:31.841 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T05:11:31.841 INFO:journalctl@ceph.mon.c.smithi086.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-02T05:11:31.841 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T05:11:31.841 INFO:journalctl@ceph.mon.c.smithi086.stdout: return future.result(timeout) 2024-04-02T05:11:31.841 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T05:11:31.842 INFO:journalctl@ceph.mon.c.smithi086.stdout: return self.__get_result() 2024-04-02T05:11:31.842 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T05:11:31.842 INFO:journalctl@ceph.mon.c.smithi086.stdout: raise self._exception 2024-04-02T05:11:31.842 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T05:11:31.842 INFO:journalctl@ceph.mon.c.smithi086.stdout: config_blobs=daemon_spec.final_config, 2024-04-02T05:11:31.842 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T05:11:31.842 INFO:journalctl@ceph.mon.c.smithi086.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T05:11:31.842 INFO:journalctl@ceph.mon.c.smithi086.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:11:31.842 INFO:journalctl@ceph.mon.c.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T05:11:31.843 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:11:31.843 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:11:31.843 INFO:journalctl@ceph.mon.c.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:11:31.843 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:31.843 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:31.843 INFO:journalctl@ceph.mon.c.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:11:31.844 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:11:31.844 INFO:journalctl@ceph.mon.c.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:11:31.844 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:11:31.844 INFO:journalctl@ceph.mon.c.smithi086.stdout: exec(code, run_globals) 2024-04-02T05:11:31.844 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:11:31.844 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:11:31.844 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:11:31.845 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:11:31.845 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:11:31.845 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:11:31.845 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:11:31.845 INFO:journalctl@ceph.mon.a.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:31.846 INFO:journalctl@ceph.mon.a.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:31.846 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:31 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:11:31.846 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:31 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:31.846 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:31 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:31.846 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:31 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:31.846 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:31 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:31.846 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:31 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:31.846 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:31 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:31.846 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:31 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:31.847 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:31 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:31.847 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:31 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:31.847 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:11:31.847 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:11:31.847 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:11:31.847 INFO:journalctl@ceph.mon.c.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:31.848 INFO:journalctl@ceph.mon.c.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:31.848 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:31 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:11:31.848 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:31 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:31.848 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:31 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:31.848 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:31 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:31.848 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:31 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:31.848 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:31 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:31.848 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:31 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:31.848 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:31 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:31.848 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:31 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:31.848 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:31 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:33.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:32 smithi086 ceph-mon[24811]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T05:11:33.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:32 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:11:33.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:32 smithi086 ceph-mon[29505]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T05:11:33.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:32 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:11:33.354 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:32 smithi192 ceph-mon[25697]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T05:11:33.354 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:32 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:11:33.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:33 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:11:33] "GET /metrics HTTP/1.1" 200 37984 "" "Prometheus/2.33.4" 2024-04-02T05:11:34.049 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:33 smithi192 ceph-mon[25697]: Upgrade: Updating prometheus.a 2024-04-02T05:11:34.049 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:33 smithi192 ceph-mon[25697]: Deploying daemon prometheus.a on smithi192 2024-04-02T05:11:34.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:33 smithi086 ceph-mon[24811]: Upgrade: Updating prometheus.a 2024-04-02T05:11:34.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:33 smithi086 ceph-mon[24811]: Deploying daemon prometheus.a on smithi192 2024-04-02T05:11:34.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:33 smithi086 ceph-mon[29505]: Upgrade: Updating prometheus.a 2024-04-02T05:11:34.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:33 smithi086 ceph-mon[29505]: Deploying daemon prometheus.a on smithi192 2024-04-02T05:11:35.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:34 smithi086 ceph-mon[24811]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T05:11:35.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:34 smithi086 ceph-mon[29505]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T05:11:35.367 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:34 smithi192 ceph-mon[25697]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T05:11:35.994 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:35 smithi192 systemd[1]: Stopping Ceph prometheus.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:11:35.994 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:35 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[60859]: ts=2024-04-02T05:11:35.978Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-02T05:11:35.994 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:35 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[60859]: ts=2024-04-02T05:11:35.979Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-02T05:11:35.994 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:35 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[60859]: ts=2024-04-02T05:11:35.979Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-02T05:11:35.994 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:35 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[60859]: ts=2024-04-02T05:11:35.979Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-02T05:11:35.994 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:35 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[60859]: ts=2024-04-02T05:11:35.979Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-02T05:11:35.994 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:35 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[60859]: ts=2024-04-02T05:11:35.979Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-02T05:11:35.994 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:35 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[60859]: ts=2024-04-02T05:11:35.979Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-02T05:11:35.995 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:35 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[60859]: ts=2024-04-02T05:11:35.979Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-02T05:11:35.995 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:35 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[60859]: ts=2024-04-02T05:11:35.979Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-02T05:11:35.995 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:35 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[60859]: ts=2024-04-02T05:11:35.979Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-02T05:11:35.995 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:35 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[60859]: ts=2024-04-02T05:11:35.979Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-02T05:11:35.995 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:35 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[60859]: ts=2024-04-02T05:11:35.979Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-02T05:11:36.249 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:35 smithi192 podman[64246]: 2024-04-02 05:11:35.989030856 +0000 UTC m=+0.117349743 container died f61f481e74750b5833c61a0e6a9ba4dc8ce0bcb5407d0e1203fe9a7c6d4eedd5 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T05:11:36.506 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:36 smithi192 podman[64246]: 2024-04-02 05:11:36.320110133 +0000 UTC m=+0.448429014 container cleanup f61f481e74750b5833c61a0e6a9ba4dc8ce0bcb5407d0e1203fe9a7c6d4eedd5 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T05:11:36.506 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:36 smithi192 bash[64246]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a 2024-04-02T05:11:36.927 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:36 smithi192 podman[64260]: 2024-04-02 05:11:36.504720543 +0000 UTC m=+0.519010327 container remove f61f481e74750b5833c61a0e6a9ba4dc8ce0bcb5407d0e1203fe9a7c6d4eedd5 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T05:11:36.927 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:36 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@prometheus.a.service: Deactivated successfully. 2024-04-02T05:11:36.927 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:36 smithi192 systemd[1]: Stopped Ceph prometheus.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:11:36.927 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:36 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@prometheus.a.service: Consumed 1.259s CPU time. 2024-04-02T05:11:37.305 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:36 smithi192 ceph-mon[25697]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 722 B/s rd, 0 op/s 2024-04-02T05:11:37.305 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:36 smithi192 systemd[1]: Starting Ceph prometheus.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:11:37.306 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:37 smithi192 podman[64392]: 2024-04-02 05:11:37.049603512 +0000 UTC m=+0.018225821 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-02T05:11:37.306 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:37 smithi192 podman[64392]: 2024-04-02 05:11:37.181382251 +0000 UTC m=+0.150004550 container create 8378fcdb60e3d8fa3b97b388d33d409f20060c23ebb9c6d438a55a8b4bf207cd (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T05:11:37.322 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:36 smithi086 ceph-mon[24811]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 722 B/s rd, 0 op/s 2024-04-02T05:11:37.322 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:36 smithi086 ceph-mon[29505]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 722 B/s rd, 0 op/s 2024-04-02T05:11:37.322 INFO:teuthology.orchestra.run.smithi086.stdout:true 2024-04-02T05:11:37.679 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:37 smithi192 podman[64392]: 2024-04-02 05:11:37.417565404 +0000 UTC m=+0.386187703 container init 8378fcdb60e3d8fa3b97b388d33d409f20060c23ebb9c6d438a55a8b4bf207cd (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T05:11:37.679 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:37 smithi192 podman[64392]: 2024-04-02 05:11:37.420103896 +0000 UTC m=+0.388726195 container start 8378fcdb60e3d8fa3b97b388d33d409f20060c23ebb9c6d438a55a8b4bf207cd (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T05:11:37.679 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:11:37.442Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-02T05:11:37.679 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:11:37.443Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-02T05:11:37.679 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:11:37.443Z caller=main.go:570 level=info host_details="(Linux 5.14.0-434.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Mar 26 19:20:47 UTC 2024 x86_64 smithi192 (none))" 2024-04-02T05:11:37.679 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:11:37.443Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-02T05:11:37.679 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:11:37.443Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-02T05:11:37.679 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:11:37.445Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-02T05:11:37.679 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:11:37.445Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-02T05:11:37.679 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:11:37.446Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-02T05:11:37.679 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:11:37.446Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-02T05:11:37.679 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:11:37.449Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-02T05:11:37.680 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:11:37.449Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.635µs 2024-04-02T05:11:37.680 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:11:37.449Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-02T05:11:37.680 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:11:37.456Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=6 2024-04-02T05:11:37.680 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:11:37.469Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=6 2024-04-02T05:11:37.680 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:11:37.476Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=6 2024-04-02T05:11:37.680 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:11:37.479Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=6 2024-04-02T05:11:37.680 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:11:37.481Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=6 2024-04-02T05:11:37.680 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:11:37.482Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=6 2024-04-02T05:11:37.680 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:11:37.482Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=6 maxSegment=6 2024-04-02T05:11:37.680 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:11:37.482Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=27.791µs wal_replay_duration=33.473475ms wbl_replay_duration=120ns total_replay_duration=33.519651ms 2024-04-02T05:11:37.680 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:11:37.485Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-02T05:11:37.680 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:11:37.485Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-02T05:11:37.680 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:11:37.485Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-02T05:11:37.680 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:37 smithi192 bash[64392]: 8378fcdb60e3d8fa3b97b388d33d409f20060c23ebb9c6d438a55a8b4bf207cd 2024-04-02T05:11:37.680 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:37 smithi192 systemd[1]: Started Ceph prometheus.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:11:37.681 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:11:37.505Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=20.103251ms db_storage=844ns remote_storage=1.155µs web_handler=349ns query_engine=713ns scrape=430.619µs scrape_sd=71.882µs notify=14.61µs notify_sd=17.457µs rules=19.347005ms tracing=7.089µs 2024-04-02T05:11:37.681 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:11:37.505Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-02T05:11:37.681 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:11:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:11:37.505Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-02T05:11:37.966 INFO:teuthology.orchestra.run.smithi086.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T05:11:37.966 INFO:teuthology.orchestra.run.smithi086.stdout:alertmanager.a smithi086 *:9093,9094 running (13m) 15s ago 21m 22.7M - 0.23.0 ba2b418f427c b2769c8e287a 2024-04-02T05:11:37.966 INFO:teuthology.orchestra.run.smithi086.stdout:grafana.a smithi192 *:3000 error 15s ago 21m - - 2024-04-02T05:11:37.966 INFO:teuthology.orchestra.run.smithi086.stdout:iscsi.foo.smithi086.rsbqwx smithi086 running (22s) 15s ago 20m 43.9M - 3.5 e1d6a67b021e d1199e668e41 2024-04-02T05:11:37.966 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.x smithi192 *:8443,9283,8765 running (45s) 15s ago 25m 448M - 19.0.0-1979-g6cec793d 78f830190839 dd1e6d0f0a95 2024-04-02T05:11:37.967 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.y smithi086 *:8443,9283,8765 running (10m) 15s ago 27m 499M - 19.0.0-1979-g6cec793d 78f830190839 3dca54a018ab 2024-04-02T05:11:37.967 INFO:teuthology.orchestra.run.smithi086.stdout:mon.a smithi086 running (27m) 15s ago 27m 77.7M 2048M 17.2.0 e1d6a67b021e b6c507ab96f8 2024-04-02T05:11:37.967 INFO:teuthology.orchestra.run.smithi086.stdout:mon.b smithi192 running (26m) 15s ago 26m 60.4M 2048M 17.2.0 e1d6a67b021e 4dadd8d1c318 2024-04-02T05:11:37.967 INFO:teuthology.orchestra.run.smithi086.stdout:mon.c smithi086 running (26m) 15s ago 26m 57.1M 2048M 17.2.0 e1d6a67b021e 49275483a2d4 2024-04-02T05:11:37.967 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.a smithi086 *:9100 running (27s) 15s ago 22m 8066k - 1.5.0 0da6a335fe13 05a844569d7b 2024-04-02T05:11:37.967 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.b smithi192 *:9100 running (19s) 15s ago 22m 7541k - 1.5.0 0da6a335fe13 1a7d6788d58b 2024-04-02T05:11:37.967 INFO:teuthology.orchestra.run.smithi086.stdout:osd.0 smithi086 running (25m) 15s ago 25m 57.5M 2992M 17.2.0 e1d6a67b021e 6d90c60d1c71 2024-04-02T05:11:37.967 INFO:teuthology.orchestra.run.smithi086.stdout:osd.1 smithi086 running (25m) 15s ago 25m 56.4M 2992M 17.2.0 e1d6a67b021e 203689f71603 2024-04-02T05:11:37.967 INFO:teuthology.orchestra.run.smithi086.stdout:osd.2 smithi086 running (24m) 15s ago 24m 50.3M 2992M 17.2.0 e1d6a67b021e 472c860a9af8 2024-04-02T05:11:37.967 INFO:teuthology.orchestra.run.smithi086.stdout:osd.3 smithi086 running (24m) 15s ago 24m 52.6M 2992M 17.2.0 e1d6a67b021e 327538dac44e 2024-04-02T05:11:37.967 INFO:teuthology.orchestra.run.smithi086.stdout:osd.4 smithi192 running (24m) 15s ago 24m 56.0M 3248M 17.2.0 e1d6a67b021e 6535d7919a2d 2024-04-02T05:11:37.967 INFO:teuthology.orchestra.run.smithi086.stdout:osd.5 smithi192 running (23m) 15s ago 23m 54.8M 3248M 17.2.0 e1d6a67b021e 7259249605f2 2024-04-02T05:11:37.967 INFO:teuthology.orchestra.run.smithi086.stdout:osd.6 smithi192 running (23m) 15s ago 23m 52.8M 3248M 17.2.0 e1d6a67b021e bb7d171b02a1 2024-04-02T05:11:37.967 INFO:teuthology.orchestra.run.smithi086.stdout:osd.7 smithi192 running (23m) 15s ago 23m 54.5M 3248M 17.2.0 e1d6a67b021e 95f5a0353999 2024-04-02T05:11:37.968 INFO:teuthology.orchestra.run.smithi086.stdout:prometheus.a smithi192 *:9095 starting - - - - 2024-04-02T05:11:37.968 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi086.bxzxlp smithi086 *:8000 running (20m) 15s ago 20m 77.7M - 17.2.0 e1d6a67b021e 6725785fbc53 2024-04-02T05:11:37.968 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi192.pmlbot smithi192 *:8000 running (20m) 15s ago 20m 77.0M - 17.2.0 e1d6a67b021e 5ae3638f8b1c 2024-04-02T05:11:38.359 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:11:38.360 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": { 2024-04-02T05:11:38.360 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-02T05:11:38.360 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:11:38.360 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": { 2024-04-02T05:11:38.360 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 2 2024-04-02T05:11:38.360 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:11:38.360 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": { 2024-04-02T05:11:38.360 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-02T05:11:38.360 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:11:38.360 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": {}, 2024-04-02T05:11:38.360 INFO:teuthology.orchestra.run.smithi086.stdout: "rgw": { 2024-04-02T05:11:38.360 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-02T05:11:38.360 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:11:38.361 INFO:teuthology.orchestra.run.smithi086.stdout: "overall": { 2024-04-02T05:11:38.361 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-02T05:11:38.361 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 2 2024-04-02T05:11:38.361 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-04-02T05:11:38.361 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:11:38.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:38 smithi192 ceph-mon[25697]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 716 B/s rd, 0 op/s 2024-04-02T05:11:38.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:38 smithi192 ceph-mon[25697]: from='client.25483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:11:38.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:38 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:38.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:38 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:38.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:38 smithi192 ceph-mon[25697]: from='client.25489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:11:38.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:38 smithi192 ceph-mon[25697]: from='client.15606 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:11:38.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:38 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/2676490600' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:38.686 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:11:38.686 INFO:teuthology.orchestra.run.smithi086.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", 2024-04-02T05:11:38.686 INFO:teuthology.orchestra.run.smithi086.stdout: "in_progress": true, 2024-04-02T05:11:38.687 INFO:teuthology.orchestra.run.smithi086.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-04-02T05:11:38.687 INFO:teuthology.orchestra.run.smithi086.stdout: "services_complete": [ 2024-04-02T05:11:38.687 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr" 2024-04-02T05:11:38.687 INFO:teuthology.orchestra.run.smithi086.stdout: ], 2024-04-02T05:11:38.687 INFO:teuthology.orchestra.run.smithi086.stdout: "progress": "2/2 daemons upgraded", 2024-04-02T05:11:38.687 INFO:teuthology.orchestra.run.smithi086.stdout: "message": "Currently upgrading prometheus daemons", 2024-04-02T05:11:38.687 INFO:teuthology.orchestra.run.smithi086.stdout: "is_paused": false 2024-04-02T05:11:38.687 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:11:38.687 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:38 smithi086 ceph-mon[24811]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 716 B/s rd, 0 op/s 2024-04-02T05:11:38.688 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:38 smithi086 ceph-mon[24811]: from='client.25483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:11:38.688 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:38 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:38.688 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:38 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:38.688 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:38 smithi086 ceph-mon[24811]: from='client.25489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:11:38.688 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:38 smithi086 ceph-mon[24811]: from='client.15606 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:11:38.688 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:38 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/2676490600' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:38.688 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:38 smithi086 ceph-mon[29505]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 716 B/s rd, 0 op/s 2024-04-02T05:11:38.689 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:38 smithi086 ceph-mon[29505]: from='client.25483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:11:38.689 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:38 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:38.689 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:38 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:38.689 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:38 smithi086 ceph-mon[29505]: from='client.25489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:11:38.689 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:38 smithi086 ceph-mon[29505]: from='client.15606 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:11:38.689 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:38 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/2676490600' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:39.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:39 smithi086 ceph-mon[29505]: from='client.25507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:11:39.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:39 smithi086 ceph-mon[24811]: from='client.25507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:11:39.924 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:39 smithi192 ceph-mon[25697]: from='client.25507 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:11:40.832 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:40 smithi086 ceph-mon[29505]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-02T05:11:40.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:40 smithi086 ceph-mon[24811]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-02T05:11:40.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:40 smithi192 ceph-mon[25697]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-04-02T05:11:42.491 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:42 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:42.491 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:42 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:42.491 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:42 smithi086 ceph-mon[29505]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-02T05:11:42.491 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:42 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:42.491 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:42 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:42.492 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:42 smithi086 ceph-mon[24811]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-02T05:11:42.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:42 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:42.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:42 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:42.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:42 smithi192 ceph-mon[25697]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-02T05:11:43.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:43 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:43.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:43 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:43.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:43 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/2826052479' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T05:11:43.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:43 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:11:43] "GET /metrics HTTP/1.1" 200 37984 "" "Prometheus/2.43.0" 2024-04-02T05:11:43.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:43 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:43.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:43 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:43.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:43 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/2826052479' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T05:11:43.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:43 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:43.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:43 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:43.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:43 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/2826052479' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T05:11:44.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:44 smithi192 ceph-mon[25697]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-02T05:11:44.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:44 smithi086 ceph-mon[24811]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-02T05:11:44.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:44 smithi086 ceph-mon[29505]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-02T05:11:46.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:46 smithi086 ceph-mon[29505]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-02T05:11:46.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:46 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:46.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:46 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:46.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:46 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:11:46.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:46 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:11:46.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:46 smithi086 ceph-mon[29505]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 199 B/s rd, 0 op/s 2024-04-02T05:11:46.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:46 smithi086 ceph-mon[29505]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 247 B/s rd, 0 op/s 2024-04-02T05:11:46.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:46 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:46.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:46 smithi086 ceph-mon[29505]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T05:11:46.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:46 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:11:46.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:46 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:11:46.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:46 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:11:46.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:46 smithi086 ceph-mon[29505]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T05:11:46.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:46 smithi086 ceph-mon[24811]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-02T05:11:46.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:46 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:46.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:46 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:46.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:46 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:11:46.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:46 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:11:46.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:46 smithi086 ceph-mon[24811]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 199 B/s rd, 0 op/s 2024-04-02T05:11:46.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:46 smithi086 ceph-mon[24811]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 247 B/s rd, 0 op/s 2024-04-02T05:11:46.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:46 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:46.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:46 smithi086 ceph-mon[24811]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T05:11:46.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:46 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:11:46.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:46 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:11:46.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:46 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:11:46.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:46 smithi086 ceph-mon[24811]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T05:11:46.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:46 smithi192 ceph-mon[25697]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-02T05:11:46.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:46 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:46.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:46 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:46.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:46 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:11:46.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:46 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:11:46.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:46 smithi192 ceph-mon[25697]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 199 B/s rd, 0 op/s 2024-04-02T05:11:46.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:46 smithi192 ceph-mon[25697]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 247 B/s rd, 0 op/s 2024-04-02T05:11:46.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:46 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:46.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:46 smithi192 ceph-mon[25697]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T05:11:46.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:46 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:11:46.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:46 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:11:46.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:46 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:11:46.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:46 smithi192 ceph-mon[25697]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T05:11:48.691 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-mon[24811]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 247 B/s rd, 0 op/s 2024-04-02T05:11:48.691 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:11:48.691 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-mon[29505]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 247 B/s rd, 0 op/s 2024-04-02T05:11:48.692 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:11:48.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:48 smithi192 ceph-mon[25697]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 247 B/s rd, 0 op/s 2024-04-02T05:11:48.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:48 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:11:49.083 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:11:48.772+0000 7f180812e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:11:49.083 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Creating ceph-iscsi config... 2024-04-02T05:11:49.084 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:11:49.084 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:11:49.084 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:11:49.084 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:49.084 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:49.084 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Traceback (most recent call last): 2024-04-02T05:11:49.084 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:11:49.084 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: return _run_code(code, main_globals, None, 2024-04-02T05:11:49.084 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:11:49.084 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: exec(code, run_globals) 2024-04-02T05:11:49.084 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:11:49.084 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:11:49.084 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:11:49.085 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:11:49.085 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:11:49.085 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:11:49.085 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:11:49.085 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:49.085 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:49.085 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Traceback (most recent call last): 2024-04-02T05:11:49.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T05:11:49.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T05:11:49.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T05:11:49.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T05:11:49.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T05:11:49.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: return self.event_loop.get_result(coro, timeout) 2024-04-02T05:11:49.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T05:11:49.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: return future.result(timeout) 2024-04-02T05:11:49.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T05:11:49.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: return self.__get_result() 2024-04-02T05:11:49.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T05:11:49.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: raise self._exception 2024-04-02T05:11:49.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T05:11:49.086 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: config_blobs=daemon_spec.final_config, 2024-04-02T05:11:49.087 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T05:11:49.087 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T05:11:49.087 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:11:49.087 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Creating ceph-iscsi config... 2024-04-02T05:11:49.087 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:11:49.087 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:11:49.087 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:11:49.087 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:49.087 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:49.087 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Traceback (most recent call last): 2024-04-02T05:11:49.087 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:11:49.087 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: return _run_code(code, main_globals, None, 2024-04-02T05:11:49.087 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:11:49.087 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: exec(code, run_globals) 2024-04-02T05:11:49.087 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:11:49.088 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:11:49.088 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:11:49.088 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:11:49.088 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:11:49.088 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:11:49.088 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:11:49.088 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:49.088 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:48 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:49.626 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:49 smithi086 ceph-mon[24811]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:11:49.627 INFO:journalctl@ceph.mon.a.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T05:11:49.627 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:11:49.627 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:11:49.627 INFO:journalctl@ceph.mon.a.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:11:49.627 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:49.627 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:49.627 INFO:journalctl@ceph.mon.a.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:11:49.627 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:11:49.627 INFO:journalctl@ceph.mon.a.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:11:49.627 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:11:49.627 INFO:journalctl@ceph.mon.a.smithi086.stdout: exec(code, run_globals) 2024-04-02T05:11:49.627 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:11:49.628 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:11:49.628 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:11:49.628 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:11:49.628 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:11:49.628 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:11:49.628 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:11:49.628 INFO:journalctl@ceph.mon.a.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:49.628 INFO:journalctl@ceph.mon.a.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:49.628 INFO:journalctl@ceph.mon.a.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:11:49.628 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T05:11:49.628 INFO:journalctl@ceph.mon.a.smithi086.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T05:11:49.628 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T05:11:49.628 INFO:journalctl@ceph.mon.a.smithi086.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T05:11:49.628 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T05:11:49.628 INFO:journalctl@ceph.mon.a.smithi086.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-02T05:11:49.629 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T05:11:49.629 INFO:journalctl@ceph.mon.a.smithi086.stdout: return future.result(timeout) 2024-04-02T05:11:49.629 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T05:11:49.629 INFO:journalctl@ceph.mon.a.smithi086.stdout: return self.__get_result() 2024-04-02T05:11:49.629 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T05:11:49.629 INFO:journalctl@ceph.mon.a.smithi086.stdout: raise self._exception 2024-04-02T05:11:49.629 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T05:11:49.629 INFO:journalctl@ceph.mon.a.smithi086.stdout: config_blobs=daemon_spec.final_config, 2024-04-02T05:11:49.629 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T05:11:49.629 INFO:journalctl@ceph.mon.a.smithi086.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T05:11:49.629 INFO:journalctl@ceph.mon.a.smithi086.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:11:49.629 INFO:journalctl@ceph.mon.a.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T05:11:49.629 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:11:49.629 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:11:49.629 INFO:journalctl@ceph.mon.a.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:11:49.630 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:49.630 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:49.630 INFO:journalctl@ceph.mon.a.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:11:49.630 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:11:49.630 INFO:journalctl@ceph.mon.a.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:11:49.630 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:11:49.630 INFO:journalctl@ceph.mon.a.smithi086.stdout: exec(code, run_globals) 2024-04-02T05:11:49.630 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:11:49.630 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:11:49.630 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:11:49.630 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:11:49.630 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:11:49.630 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:11:49.630 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:11:49.631 INFO:journalctl@ceph.mon.a.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:49.631 INFO:journalctl@ceph.mon.a.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:49.631 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:49 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T05:11:49.631 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:49 smithi086 ceph-mon[24811]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T05:11:49.631 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:49 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:11:49.631 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:49 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:49.631 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:49 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:49.631 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:49 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:49.631 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:49 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:49.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:49 smithi192 ceph-mon[25697]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:11:49.927 INFO:journalctl@ceph.mon.b.smithi192.stdout: Creating ceph-iscsi config... 2024-04-02T05:11:49.927 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:11:49.928 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:11:49.928 INFO:journalctl@ceph.mon.b.smithi192.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:11:49.928 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:49.928 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:49.928 INFO:journalctl@ceph.mon.b.smithi192.stdout: Traceback (most recent call last): 2024-04-02T05:11:49.928 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:11:49.928 INFO:journalctl@ceph.mon.b.smithi192.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:11:49.928 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:11:49.928 INFO:journalctl@ceph.mon.b.smithi192.stdout: exec(code, run_globals) 2024-04-02T05:11:49.928 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:11:49.928 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:11:49.928 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:11:49.928 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:11:49.928 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:11:49.929 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:11:49.929 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:11:49.929 INFO:journalctl@ceph.mon.b.smithi192.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:49.929 INFO:journalctl@ceph.mon.b.smithi192.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:49.929 INFO:journalctl@ceph.mon.b.smithi192.stdout: Traceback (most recent call last): 2024-04-02T05:11:49.929 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T05:11:49.929 INFO:journalctl@ceph.mon.b.smithi192.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T05:11:49.929 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T05:11:49.929 INFO:journalctl@ceph.mon.b.smithi192.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T05:11:49.929 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T05:11:49.929 INFO:journalctl@ceph.mon.b.smithi192.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-02T05:11:49.929 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T05:11:49.929 INFO:journalctl@ceph.mon.b.smithi192.stdout: return future.result(timeout) 2024-04-02T05:11:49.930 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T05:11:49.930 INFO:journalctl@ceph.mon.b.smithi192.stdout: return self.__get_result() 2024-04-02T05:11:49.930 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T05:11:49.930 INFO:journalctl@ceph.mon.b.smithi192.stdout: raise self._exception 2024-04-02T05:11:49.930 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T05:11:49.930 INFO:journalctl@ceph.mon.b.smithi192.stdout: config_blobs=daemon_spec.final_config, 2024-04-02T05:11:49.930 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T05:11:49.930 INFO:journalctl@ceph.mon.b.smithi192.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T05:11:49.930 INFO:journalctl@ceph.mon.b.smithi192.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:11:49.930 INFO:journalctl@ceph.mon.b.smithi192.stdout: Creating ceph-iscsi config... 2024-04-02T05:11:49.930 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:11:49.930 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:11:49.930 INFO:journalctl@ceph.mon.b.smithi192.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:11:49.930 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:49.931 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:49.931 INFO:journalctl@ceph.mon.b.smithi192.stdout: Traceback (most recent call last): 2024-04-02T05:11:49.931 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:11:49.931 INFO:journalctl@ceph.mon.b.smithi192.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:11:49.931 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:11:49.931 INFO:journalctl@ceph.mon.b.smithi192.stdout: exec(code, run_globals) 2024-04-02T05:11:49.931 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:11:49.931 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:11:49.931 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:11:49.931 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:11:49.931 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:11:49.931 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:11:49.931 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:11:49.932 INFO:journalctl@ceph.mon.b.smithi192.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:49.932 INFO:journalctl@ceph.mon.b.smithi192.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:49.932 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:49 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T05:11:49.932 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:49 smithi192 ceph-mon[25697]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T05:11:49.932 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:49 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:11:49.932 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:49 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:49.932 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:49 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:49.932 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:49 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:49.932 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:49 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:49.933 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:49 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:49.933 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:49 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:49.933 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:49 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:49.933 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:49 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:49.933 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:49 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:49.933 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:49 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:49.989 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:49 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:49.989 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:49 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:49.989 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:49 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:49.989 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:49 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:49.989 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:49 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:49.989 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:49 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:49.990 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:49 smithi086 ceph-mon[29505]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:11:49.990 INFO:journalctl@ceph.mon.c.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T05:11:49.990 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:11:49.990 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:11:49.990 INFO:journalctl@ceph.mon.c.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:11:49.990 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:49.990 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:49.990 INFO:journalctl@ceph.mon.c.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:11:49.990 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:11:49.990 INFO:journalctl@ceph.mon.c.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:11:49.990 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:11:49.991 INFO:journalctl@ceph.mon.c.smithi086.stdout: exec(code, run_globals) 2024-04-02T05:11:49.991 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:11:49.991 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:11:49.991 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:11:49.991 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:11:49.991 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:11:49.991 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:11:49.991 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:11:49.991 INFO:journalctl@ceph.mon.c.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:49.991 INFO:journalctl@ceph.mon.c.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:49.991 INFO:journalctl@ceph.mon.c.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:11:49.991 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T05:11:49.991 INFO:journalctl@ceph.mon.c.smithi086.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T05:11:49.991 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T05:11:49.992 INFO:journalctl@ceph.mon.c.smithi086.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T05:11:49.992 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T05:11:49.992 INFO:journalctl@ceph.mon.c.smithi086.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-02T05:11:49.992 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T05:11:49.992 INFO:journalctl@ceph.mon.c.smithi086.stdout: return future.result(timeout) 2024-04-02T05:11:49.992 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T05:11:49.992 INFO:journalctl@ceph.mon.c.smithi086.stdout: return self.__get_result() 2024-04-02T05:11:49.992 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T05:11:49.992 INFO:journalctl@ceph.mon.c.smithi086.stdout: raise self._exception 2024-04-02T05:11:49.992 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T05:11:49.992 INFO:journalctl@ceph.mon.c.smithi086.stdout: config_blobs=daemon_spec.final_config, 2024-04-02T05:11:49.992 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T05:11:49.992 INFO:journalctl@ceph.mon.c.smithi086.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T05:11:49.992 INFO:journalctl@ceph.mon.c.smithi086.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:11:49.993 INFO:journalctl@ceph.mon.c.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T05:11:49.993 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:11:49.993 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:11:49.993 INFO:journalctl@ceph.mon.c.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:11:49.993 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:49.993 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:49.993 INFO:journalctl@ceph.mon.c.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:11:49.993 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:11:49.993 INFO:journalctl@ceph.mon.c.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:11:49.993 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:11:49.993 INFO:journalctl@ceph.mon.c.smithi086.stdout: exec(code, run_globals) 2024-04-02T05:11:49.993 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:11:49.993 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:11:49.993 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:11:49.994 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:11:49.994 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:11:49.994 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:11:49.994 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:11:49.994 INFO:journalctl@ceph.mon.c.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:11:49.994 INFO:journalctl@ceph.mon.c.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:11:49.994 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:49 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T05:11:49.994 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:49 smithi086 ceph-mon[29505]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T05:11:49.994 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:49 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:11:49.994 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:49 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:49.994 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:49 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:49.994 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:49 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:49.994 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:49 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:49.995 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:49 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:49.995 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:49 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:49.995 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:49 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:49.995 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:49 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:49.995 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:49 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:49.995 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:49 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:11:50.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:50 smithi192 ceph-mon[25697]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 867 B/s rd, 1 op/s 2024-04-02T05:11:50.983 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:50 smithi086 ceph-mon[24811]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 867 B/s rd, 1 op/s 2024-04-02T05:11:50.984 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:50 smithi086 ceph-mon[29505]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 867 B/s rd, 1 op/s 2024-04-02T05:11:52.265 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:51 smithi086 ceph-mon[24811]: Upgrade: Updating alertmanager.a 2024-04-02T05:11:52.265 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:51 smithi086 ceph-mon[24811]: Deploying daemon alertmanager.a on smithi086 2024-04-02T05:11:52.265 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:51 smithi086 ceph-mon[29505]: Upgrade: Updating alertmanager.a 2024-04-02T05:11:52.265 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:51 smithi086 ceph-mon[29505]: Deploying daemon alertmanager.a on smithi086 2024-04-02T05:11:52.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:51 smithi192 ceph-mon[25697]: Upgrade: Updating alertmanager.a 2024-04-02T05:11:52.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:51 smithi192 ceph-mon[25697]: Deploying daemon alertmanager.a on smithi086 2024-04-02T05:11:53.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:52 smithi086 ceph-mon[24811]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 867 B/s rd, 1 op/s 2024-04-02T05:11:53.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:52 smithi086 ceph-mon[29505]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 867 B/s rd, 1 op/s 2024-04-02T05:11:53.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:52 smithi192 ceph-mon[25697]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 867 B/s rd, 1 op/s 2024-04-02T05:11:53.814 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:11:53 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:11:53] "GET /metrics HTTP/1.1" 200 37981 "" "Prometheus/2.43.0" 2024-04-02T05:11:54.065 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:11:53 smithi086 systemd[1]: Stopping Ceph alertmanager.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:11:54.333 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:11:54 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[59010]: level=info ts=2024-04-02T05:11:54.116Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-04-02T05:11:54.333 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:11:54 smithi086 podman[79753]: 2024-04-02 05:11:54.167679856 +0000 UTC m=+0.187787856 container died b2769c8e287a443c0b55b1ad86e7a1f48760f11038e4d4dc2c76557f77a58b52 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T05:11:54.821 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:11:54 smithi086 podman[79753]: 2024-04-02 05:11:54.550470573 +0000 UTC m=+0.570578572 container cleanup b2769c8e287a443c0b55b1ad86e7a1f48760f11038e4d4dc2c76557f77a58b52 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T05:11:54.821 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:11:54 smithi086 bash[79753]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a 2024-04-02T05:11:55.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:54 smithi086 ceph-mon[24811]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 619 B/s rd, 0 op/s 2024-04-02T05:11:55.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:54 smithi086 ceph-mon[29505]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 619 B/s rd, 0 op/s 2024-04-02T05:11:55.084 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:11:54 smithi086 podman[79770]: 2024-04-02 05:11:54.818270719 +0000 UTC m=+0.655244153 container remove b2769c8e287a443c0b55b1ad86e7a1f48760f11038e4d4dc2c76557f77a58b52 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T05:11:55.084 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:11:54 smithi086 podman[79770]: 2024-04-02 05:11:54.893926882 +0000 UTC m=+0.730900315 volume remove eb75ff9da27b94fb3cdbf822094e5a0f0f27572972d9a273c6273ac00c3d617a 2024-04-02T05:11:55.084 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:11:55 smithi086 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@alertmanager.a.service: Deactivated successfully. 2024-04-02T05:11:55.084 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:11:55 smithi086 systemd[1]: Stopped Ceph alertmanager.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:11:55.084 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:11:55 smithi086 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@alertmanager.a.service: Consumed 1.566s CPU time. 2024-04-02T05:11:55.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:54 smithi192 ceph-mon[25697]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 619 B/s rd, 0 op/s 2024-04-02T05:11:55.556 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:11:55 smithi086 systemd[1]: Starting Ceph alertmanager.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:11:55.556 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:11:55 smithi086 podman[79908]: 2024-04-02 05:11:55.445104579 +0000 UTC m=+0.111108268 volume create 3bada234eed676398a8e78b3b95a18f2c4b3e6f59da5c78155f2cd6758bd17ca 2024-04-02T05:11:55.556 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:11:55 smithi086 podman[79908]: 2024-04-02 05:11:55.352023111 +0000 UTC m=+0.018026814 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-04-02T05:11:55.556 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:11:55 smithi086 podman[79908]: 2024-04-02 05:11:55.553953331 +0000 UTC m=+0.219957020 container create 660ea62bb6b98412a61f27d504bb177e33961057c05751b11bdbf6eab8043b60 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T05:11:56.333 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:11:55 smithi086 podman[79908]: 2024-04-02 05:11:55.84849286 +0000 UTC m=+0.514496548 container init 660ea62bb6b98412a61f27d504bb177e33961057c05751b11bdbf6eab8043b60 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T05:11:56.333 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:11:55 smithi086 podman[79908]: 2024-04-02 05:11:55.851402015 +0000 UTC m=+0.517405704 container start 660ea62bb6b98412a61f27d504bb177e33961057c05751b11bdbf6eab8043b60 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T05:11:56.333 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:11:55 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[79924]: ts=2024-04-02T05:11:55.864Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-04-02T05:11:56.333 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:11:55 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[79924]: ts=2024-04-02T05:11:55.864Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-04-02T05:11:56.334 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:11:55 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[79924]: ts=2024-04-02T05:11:55.865Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.86 port=9094 2024-04-02T05:11:56.334 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:11:55 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[79924]: ts=2024-04-02T05:11:55.866Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-02T05:11:56.334 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:11:55 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[79924]: ts=2024-04-02T05:11:55.891Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-02T05:11:56.334 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:11:55 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[79924]: ts=2024-04-02T05:11:55.891Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-02T05:11:56.334 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:11:55 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[79924]: ts=2024-04-02T05:11:55.894Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-04-02T05:11:56.334 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:11:55 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[79924]: ts=2024-04-02T05:11:55.894Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-04-02T05:11:56.334 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:11:55 smithi086 bash[79908]: 660ea62bb6b98412a61f27d504bb177e33961057c05751b11bdbf6eab8043b60 2024-04-02T05:11:56.334 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:11:55 smithi086 systemd[1]: Started Ceph alertmanager.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:11:57.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:57 smithi086 ceph-mon[24811]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-02T05:11:57.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:57 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:57.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:57 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:57.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:57 smithi086 ceph-mon[29505]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-02T05:11:57.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:57 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:57.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:57 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:57.094 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:57 smithi192 ceph-mon[25697]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-02T05:11:57.095 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:57 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:57.095 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:57 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:11:58.333 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:11:57 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[79924]: ts=2024-04-02T05:11:57.867Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000584482s 2024-04-02T05:11:59.180 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:11:59 smithi086 ceph-mon[29505]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-02T05:11:59.180 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:11:59 smithi086 ceph-mon[24811]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-02T05:11:59.281 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:11:59 smithi192 ceph-mon[25697]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-02T05:12:01.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:00 smithi086 ceph-mon[29505]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-02T05:12:01.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:00 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:01.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:00 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:01.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:00 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:01.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:00 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:01.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:00 smithi086 ceph-mon[24811]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-02T05:12:01.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:00 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:01.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:00 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:01.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:00 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:01.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:00 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:01.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:00 smithi192 ceph-mon[25697]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-02T05:12:01.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:00 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:01.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:00 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:01.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:00 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:01.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:00 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:02.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:01 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/2986915170' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T05:12:02.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:01 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:02.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:01 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:02.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:01 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/2986915170' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T05:12:02.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:01 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:02.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:01 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:02.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:01 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/2986915170' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T05:12:02.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:01 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:02.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:01 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:03.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:02 smithi086 ceph-mon[29505]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:12:03.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:02 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:12:03.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:02 smithi086 ceph-mon[24811]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:12:03.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:02 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:12:03.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:02 smithi192 ceph-mon[25697]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:12:03.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:02 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:12:03.544 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:12:03] "GET /metrics HTTP/1.1" 200 37983 "" "Prometheus/2.43.0" 2024-04-02T05:12:05.293 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:04 smithi086 ceph-mon[24811]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:12:05.294 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:04 smithi086 ceph-mon[29505]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:12:05.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:04 smithi192 ceph-mon[25697]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:12:06.193 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:12:05 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[79924]: ts=2024-04-02T05:12:05.868Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.001915031s 2024-04-02T05:12:06.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:06 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:06.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:06 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:06.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:06 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:12:06.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:06 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:12:06.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:06 smithi086 ceph-mon[24811]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 260 B/s rd, 0 op/s 2024-04-02T05:12:06.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:06 smithi086 ceph-mon[24811]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 313 B/s rd, 0 op/s 2024-04-02T05:12:06.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:06 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:06.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:06 smithi086 ceph-mon[24811]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T05:12:06.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:06 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:12:06.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:06 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:12:06.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:06 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:12:06.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:06 smithi086 ceph-mon[24811]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T05:12:06.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:06 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:06.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:06 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:06.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:06 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:12:06.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:06 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:12:06.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:06 smithi086 ceph-mon[29505]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 260 B/s rd, 0 op/s 2024-04-02T05:12:06.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:06 smithi086 ceph-mon[29505]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 313 B/s rd, 0 op/s 2024-04-02T05:12:06.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:06 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:06.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:06 smithi086 ceph-mon[29505]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T05:12:06.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:06 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:12:06.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:06 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:12:06.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:06 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:12:06.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:06 smithi086 ceph-mon[29505]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T05:12:06.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:06 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:06.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:06 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:06.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:06 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:12:06.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:06 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:12:06.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:06 smithi192 ceph-mon[25697]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 260 B/s rd, 0 op/s 2024-04-02T05:12:06.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:06 smithi192 ceph-mon[25697]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 313 B/s rd, 0 op/s 2024-04-02T05:12:06.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:06 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:06.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:06 smithi192 ceph-mon[25697]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T05:12:06.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:06 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:12:06.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:06 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:12:06.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:06 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:12:06.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:06 smithi192 ceph-mon[25697]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T05:12:07.628 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:07 smithi086 ceph-mon[24811]: Health check update: 2 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-02T05:12:07.628 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:07 smithi086 ceph-mon[29505]: Health check update: 2 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-02T05:12:07.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:07 smithi192 ceph-mon[25697]: Health check update: 2 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-02T05:12:08.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-mon[24811]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 313 B/s rd, 0 op/s 2024-04-02T05:12:08.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-mon[29505]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 313 B/s rd, 0 op/s 2024-04-02T05:12:08.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:08 smithi192 ceph-mon[25697]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 313 B/s rd, 0 op/s 2024-04-02T05:12:09.047 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:12:08.835+0000 7f180812e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:12:09.047 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Creating ceph-iscsi config... 2024-04-02T05:12:09.047 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:12:09.047 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:12:09.047 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:12:09.048 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:12:09.048 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:12:09.048 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Traceback (most recent call last): 2024-04-02T05:12:09.048 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:12:09.048 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: return _run_code(code, main_globals, None, 2024-04-02T05:12:09.048 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:12:09.048 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: exec(code, run_globals) 2024-04-02T05:12:09.048 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:12:09.048 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:12:09.048 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:12:09.048 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:12:09.048 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:12:09.048 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:12:09.049 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:12:09.049 INFO:teuthology.orchestra.run.smithi086.stdout:true 2024-04-02T05:12:09.049 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:12:09.049 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:12:09.050 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Traceback (most recent call last): 2024-04-02T05:12:09.050 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T05:12:09.050 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T05:12:09.050 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T05:12:09.050 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T05:12:09.050 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T05:12:09.050 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: return self.event_loop.get_result(coro, timeout) 2024-04-02T05:12:09.050 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T05:12:09.050 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: return future.result(timeout) 2024-04-02T05:12:09.050 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T05:12:09.050 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: return self.__get_result() 2024-04-02T05:12:09.050 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T05:12:09.050 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: raise self._exception 2024-04-02T05:12:09.051 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T05:12:09.051 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: config_blobs=daemon_spec.final_config, 2024-04-02T05:12:09.051 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T05:12:09.051 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T05:12:09.051 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:12:09.051 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Creating ceph-iscsi config... 2024-04-02T05:12:09.051 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:12:09.051 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:12:09.051 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:12:09.051 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:12:09.051 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:12:09.051 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Traceback (most recent call last): 2024-04-02T05:12:09.051 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:12:09.051 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: return _run_code(code, main_globals, None, 2024-04-02T05:12:09.052 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:12:09.052 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: exec(code, run_globals) 2024-04-02T05:12:09.052 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:12:09.052 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:12:09.052 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:12:09.052 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:12:09.052 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:12:09.052 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:12:09.052 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:12:09.052 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:12:09.052 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:12:09.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:09 smithi192 ceph-mon[25697]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:12:09.678 INFO:journalctl@ceph.mon.b.smithi192.stdout: Creating ceph-iscsi config... 2024-04-02T05:12:09.678 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:12:09.678 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:12:09.678 INFO:journalctl@ceph.mon.b.smithi192.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:12:09.678 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:12:09.678 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:12:09.678 INFO:journalctl@ceph.mon.b.smithi192.stdout: Traceback (most recent call last): 2024-04-02T05:12:09.678 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:12:09.678 INFO:journalctl@ceph.mon.b.smithi192.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:12:09.678 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:12:09.678 INFO:journalctl@ceph.mon.b.smithi192.stdout: exec(code, run_globals) 2024-04-02T05:12:09.678 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:12:09.678 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:12:09.679 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:12:09.679 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:12:09.679 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:12:09.679 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:12:09.679 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:12:09.679 INFO:journalctl@ceph.mon.b.smithi192.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:12:09.679 INFO:journalctl@ceph.mon.b.smithi192.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:12:09.679 INFO:journalctl@ceph.mon.b.smithi192.stdout: Traceback (most recent call last): 2024-04-02T05:12:09.679 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T05:12:09.679 INFO:journalctl@ceph.mon.b.smithi192.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T05:12:09.679 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T05:12:09.679 INFO:journalctl@ceph.mon.b.smithi192.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T05:12:09.679 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T05:12:09.679 INFO:journalctl@ceph.mon.b.smithi192.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-02T05:12:09.679 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T05:12:09.680 INFO:journalctl@ceph.mon.b.smithi192.stdout: return future.result(timeout) 2024-04-02T05:12:09.680 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T05:12:09.680 INFO:journalctl@ceph.mon.b.smithi192.stdout: return self.__get_result() 2024-04-02T05:12:09.680 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T05:12:09.680 INFO:journalctl@ceph.mon.b.smithi192.stdout: raise self._exception 2024-04-02T05:12:09.680 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T05:12:09.680 INFO:journalctl@ceph.mon.b.smithi192.stdout: config_blobs=daemon_spec.final_config, 2024-04-02T05:12:09.680 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T05:12:09.680 INFO:journalctl@ceph.mon.b.smithi192.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T05:12:09.680 INFO:journalctl@ceph.mon.b.smithi192.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:12:09.680 INFO:journalctl@ceph.mon.b.smithi192.stdout: Creating ceph-iscsi config... 2024-04-02T05:12:09.680 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:12:09.680 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:12:09.680 INFO:journalctl@ceph.mon.b.smithi192.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:12:09.680 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:12:09.681 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:12:09.681 INFO:journalctl@ceph.mon.b.smithi192.stdout: Traceback (most recent call last): 2024-04-02T05:12:09.681 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:12:09.681 INFO:journalctl@ceph.mon.b.smithi192.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:12:09.681 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:12:09.681 INFO:journalctl@ceph.mon.b.smithi192.stdout: exec(code, run_globals) 2024-04-02T05:12:09.681 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:12:09.681 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:12:09.681 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:12:09.681 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:12:09.681 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:12:09.681 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:12:09.681 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:12:09.681 INFO:journalctl@ceph.mon.b.smithi192.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:12:09.682 INFO:journalctl@ceph.mon.b.smithi192.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:12:09.682 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:09 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T05:12:09.682 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:09 smithi192 ceph-mon[25697]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T05:12:09.682 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:09 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:12:09.682 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:09 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:09.682 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:09 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:09.682 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:09 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:09.682 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:09 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:09.682 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:09 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:09.682 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:09 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:09.682 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:09 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:09.682 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:09 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:09.682 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:09 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:09.683 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:09 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:09.683 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:09 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:09.685 INFO:teuthology.orchestra.run.smithi086.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T05:12:09.685 INFO:teuthology.orchestra.run.smithi086.stdout:alertmanager.a smithi086 *:9093,9094 running (14s) 8s ago 22m 15.1M - 0.25.0 c8568f914cd2 660ea62bb6b9 2024-04-02T05:12:09.685 INFO:teuthology.orchestra.run.smithi086.stdout:grafana.a smithi192 *:3000 error 9s ago 21m - - 2024-04-02T05:12:09.685 INFO:teuthology.orchestra.run.smithi086.stdout:iscsi.foo.smithi086.rsbqwx smithi086 unknown 8s ago 21m - - 2024-04-02T05:12:09.686 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.x smithi192 *:8443,9283,8765 running (77s) 9s ago 26m 449M - 19.0.0-1979-g6cec793d 78f830190839 dd1e6d0f0a95 2024-04-02T05:12:09.686 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.y smithi086 *:8443,9283,8765 running (10m) 8s ago 28m 502M - 19.0.0-1979-g6cec793d 78f830190839 3dca54a018ab 2024-04-02T05:12:09.686 INFO:teuthology.orchestra.run.smithi086.stdout:mon.a smithi086 running (28m) 8s ago 28m 81.5M 2048M 17.2.0 e1d6a67b021e b6c507ab96f8 2024-04-02T05:12:09.686 INFO:teuthology.orchestra.run.smithi086.stdout:mon.b smithi192 running (26m) 9s ago 26m 64.7M 2048M 17.2.0 e1d6a67b021e 4dadd8d1c318 2024-04-02T05:12:09.686 INFO:teuthology.orchestra.run.smithi086.stdout:mon.c smithi086 running (26m) 8s ago 26m 61.1M 2048M 17.2.0 e1d6a67b021e 49275483a2d4 2024-04-02T05:12:09.686 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.a smithi086 *:9100 running (58s) 8s ago 22m 7968k - 1.5.0 0da6a335fe13 05a844569d7b 2024-04-02T05:12:09.686 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.b smithi192 *:9100 running (51s) 9s ago 22m 15.1M - 1.5.0 0da6a335fe13 1a7d6788d58b 2024-04-02T05:12:09.686 INFO:teuthology.orchestra.run.smithi086.stdout:osd.0 smithi086 running (26m) 8s ago 26m 57.5M 2992M 17.2.0 e1d6a67b021e 6d90c60d1c71 2024-04-02T05:12:09.686 INFO:teuthology.orchestra.run.smithi086.stdout:osd.1 smithi086 running (25m) 8s ago 25m 56.4M 2992M 17.2.0 e1d6a67b021e 203689f71603 2024-04-02T05:12:09.687 INFO:teuthology.orchestra.run.smithi086.stdout:osd.2 smithi086 running (25m) 8s ago 25m 50.3M 2992M 17.2.0 e1d6a67b021e 472c860a9af8 2024-04-02T05:12:09.687 INFO:teuthology.orchestra.run.smithi086.stdout:osd.3 smithi086 running (25m) 8s ago 25m 52.6M 2992M 17.2.0 e1d6a67b021e 327538dac44e 2024-04-02T05:12:09.687 INFO:teuthology.orchestra.run.smithi086.stdout:osd.4 smithi192 running (24m) 9s ago 24m 56.0M 3248M 17.2.0 e1d6a67b021e 6535d7919a2d 2024-04-02T05:12:09.687 INFO:teuthology.orchestra.run.smithi086.stdout:osd.5 smithi192 running (24m) 9s ago 24m 54.8M 3248M 17.2.0 e1d6a67b021e 7259249605f2 2024-04-02T05:12:09.687 INFO:teuthology.orchestra.run.smithi086.stdout:osd.6 smithi192 running (23m) 9s ago 23m 52.8M 3248M 17.2.0 e1d6a67b021e bb7d171b02a1 2024-04-02T05:12:09.687 INFO:teuthology.orchestra.run.smithi086.stdout:osd.7 smithi192 running (23m) 9s ago 23m 54.5M 3248M 17.2.0 e1d6a67b021e 95f5a0353999 2024-04-02T05:12:09.687 INFO:teuthology.orchestra.run.smithi086.stdout:prometheus.a smithi192 *:9095 running (32s) 9s ago 22m 48.2M - 2.43.0 a07b618ecd1d 8378fcdb60e3 2024-04-02T05:12:09.687 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi086.bxzxlp smithi086 *:8000 running (21m) 8s ago 21m 77.9M - 17.2.0 e1d6a67b021e 6725785fbc53 2024-04-02T05:12:09.687 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi192.pmlbot smithi192 *:8000 running (21m) 9s ago 21m 77.1M - 17.2.0 e1d6a67b021e 5ae3638f8b1c 2024-04-02T05:12:09.688 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:09 smithi086 ceph-mon[24811]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:12:09.688 INFO:journalctl@ceph.mon.a.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T05:12:09.688 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:12:09.688 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:12:09.688 INFO:journalctl@ceph.mon.a.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:12:09.688 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:12:09.688 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:12:09.688 INFO:journalctl@ceph.mon.a.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:12:09.688 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:12:09.688 INFO:journalctl@ceph.mon.a.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:12:09.688 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:12:09.688 INFO:journalctl@ceph.mon.a.smithi086.stdout: exec(code, run_globals) 2024-04-02T05:12:09.689 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:12:09.689 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:12:09.689 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:12:09.689 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:12:09.689 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:12:09.689 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:12:09.689 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:12:09.689 INFO:journalctl@ceph.mon.a.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:12:09.689 INFO:journalctl@ceph.mon.a.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:12:09.689 INFO:journalctl@ceph.mon.a.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:12:09.689 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T05:12:09.689 INFO:journalctl@ceph.mon.a.smithi086.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T05:12:09.690 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:09 smithi086 ceph-mon[29505]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:12:09.690 INFO:journalctl@ceph.mon.c.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T05:12:09.690 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:12:09.690 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:12:09.690 INFO:journalctl@ceph.mon.c.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:12:09.690 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:12:09.690 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:12:09.690 INFO:journalctl@ceph.mon.c.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:12:09.690 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:12:09.690 INFO:journalctl@ceph.mon.c.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:12:09.690 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:12:09.690 INFO:journalctl@ceph.mon.c.smithi086.stdout: exec(code, run_globals) 2024-04-02T05:12:09.690 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:12:09.691 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:12:09.691 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:12:09.691 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:12:09.691 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:12:09.691 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:12:09.691 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:12:09.691 INFO:journalctl@ceph.mon.c.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:12:09.691 INFO:journalctl@ceph.mon.c.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:12:09.691 INFO:journalctl@ceph.mon.c.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:12:09.691 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T05:12:09.691 INFO:journalctl@ceph.mon.c.smithi086.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T05:12:09.692 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T05:12:09.692 INFO:journalctl@ceph.mon.a.smithi086.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T05:12:09.692 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T05:12:09.692 INFO:journalctl@ceph.mon.a.smithi086.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-02T05:12:09.692 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T05:12:09.692 INFO:journalctl@ceph.mon.a.smithi086.stdout: return future.result(timeout) 2024-04-02T05:12:09.692 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T05:12:09.692 INFO:journalctl@ceph.mon.a.smithi086.stdout: return self.__get_result() 2024-04-02T05:12:09.692 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T05:12:09.692 INFO:journalctl@ceph.mon.a.smithi086.stdout: raise self._exception 2024-04-02T05:12:09.692 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T05:12:09.692 INFO:journalctl@ceph.mon.a.smithi086.stdout: config_blobs=daemon_spec.final_config, 2024-04-02T05:12:09.692 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T05:12:09.692 INFO:journalctl@ceph.mon.a.smithi086.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T05:12:09.693 INFO:journalctl@ceph.mon.a.smithi086.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:12:09.693 INFO:journalctl@ceph.mon.a.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T05:12:09.693 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:12:09.693 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:12:09.693 INFO:journalctl@ceph.mon.a.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:12:09.693 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:12:09.693 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:12:09.693 INFO:journalctl@ceph.mon.a.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:12:09.693 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:12:09.693 INFO:journalctl@ceph.mon.a.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:12:09.693 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:12:09.693 INFO:journalctl@ceph.mon.a.smithi086.stdout: exec(code, run_globals) 2024-04-02T05:12:09.693 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:12:09.693 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:12:09.694 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:12:09.694 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:12:09.694 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T05:12:09.694 INFO:journalctl@ceph.mon.c.smithi086.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T05:12:09.694 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T05:12:09.695 INFO:journalctl@ceph.mon.c.smithi086.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-02T05:12:09.695 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T05:12:09.695 INFO:journalctl@ceph.mon.c.smithi086.stdout: return future.result(timeout) 2024-04-02T05:12:09.695 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T05:12:09.695 INFO:journalctl@ceph.mon.c.smithi086.stdout: return self.__get_result() 2024-04-02T05:12:09.695 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T05:12:09.695 INFO:journalctl@ceph.mon.c.smithi086.stdout: raise self._exception 2024-04-02T05:12:09.695 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T05:12:09.695 INFO:journalctl@ceph.mon.c.smithi086.stdout: config_blobs=daemon_spec.final_config, 2024-04-02T05:12:09.695 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T05:12:09.695 INFO:journalctl@ceph.mon.c.smithi086.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T05:12:09.696 INFO:journalctl@ceph.mon.c.smithi086.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:12:09.696 INFO:journalctl@ceph.mon.c.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T05:12:09.696 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:12:09.696 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:12:09.696 INFO:journalctl@ceph.mon.c.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:12:09.696 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:12:09.696 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:12:09.696 INFO:journalctl@ceph.mon.c.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:12:09.696 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:12:09.696 INFO:journalctl@ceph.mon.c.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:12:09.696 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:12:09.696 INFO:journalctl@ceph.mon.c.smithi086.stdout: exec(code, run_globals) 2024-04-02T05:12:09.696 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:12:09.696 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:12:09.696 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:12:09.697 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:12:09.697 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:12:09.697 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:12:09.697 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:12:09.697 INFO:journalctl@ceph.mon.c.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:12:09.697 INFO:journalctl@ceph.mon.c.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:12:09.697 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:09 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T05:12:09.697 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:09 smithi086 ceph-mon[29505]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T05:12:09.697 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:09 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:12:09.697 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:09 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:09.697 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:09 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:09.697 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:09 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:09.697 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:09 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:09.697 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:09 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:09.697 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:09 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:09.698 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:09 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:09.698 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:09 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:09.698 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:09 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:09.698 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:09 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:09.698 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:09 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:09.698 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:12:09.698 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:12:09.698 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:12:09.698 INFO:journalctl@ceph.mon.a.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:12:09.698 INFO:journalctl@ceph.mon.a.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:12:09.698 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:09 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T05:12:09.699 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:09 smithi086 ceph-mon[24811]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T05:12:09.699 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:09 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:12:09.699 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:09 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:09.699 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:09 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:09.699 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:09 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:09.699 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:09 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:09.699 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:09 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:09.699 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:09 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:09.699 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:09 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:09.699 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:09 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:09.699 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:09 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:09.699 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:09 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:09.699 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:09 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:10.071 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:12:10.071 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": { 2024-04-02T05:12:10.071 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-02T05:12:10.072 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:12:10.072 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": { 2024-04-02T05:12:10.072 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 2 2024-04-02T05:12:10.072 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:12:10.072 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": { 2024-04-02T05:12:10.072 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-02T05:12:10.072 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:12:10.072 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": {}, 2024-04-02T05:12:10.072 INFO:teuthology.orchestra.run.smithi086.stdout: "rgw": { 2024-04-02T05:12:10.072 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-02T05:12:10.072 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:12:10.072 INFO:teuthology.orchestra.run.smithi086.stdout: "overall": { 2024-04-02T05:12:10.073 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-02T05:12:10.073 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 2 2024-04-02T05:12:10.073 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-04-02T05:12:10.073 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:12:10.400 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:12:10.400 INFO:teuthology.orchestra.run.smithi086.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", 2024-04-02T05:12:10.400 INFO:teuthology.orchestra.run.smithi086.stdout: "in_progress": true, 2024-04-02T05:12:10.400 INFO:teuthology.orchestra.run.smithi086.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-04-02T05:12:10.400 INFO:teuthology.orchestra.run.smithi086.stdout: "services_complete": [ 2024-04-02T05:12:10.401 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr" 2024-04-02T05:12:10.401 INFO:teuthology.orchestra.run.smithi086.stdout: ], 2024-04-02T05:12:10.401 INFO:teuthology.orchestra.run.smithi086.stdout: "progress": "2/2 daemons upgraded", 2024-04-02T05:12:10.401 INFO:teuthology.orchestra.run.smithi086.stdout: "message": "Currently upgrading grafana daemons", 2024-04-02T05:12:10.401 INFO:teuthology.orchestra.run.smithi086.stdout: "is_paused": false 2024-04-02T05:12:10.401 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:12:10.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:10 smithi192 ceph-mon[25697]: from='client.15672 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:12:10.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:10 smithi192 ceph-mon[25697]: from='client.15678 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:12:10.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:10 smithi192 ceph-mon[25697]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 835 B/s rd, 1 op/s 2024-04-02T05:12:10.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:10 smithi192 ceph-mon[25697]: from='client.25573 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:12:10.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:10 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1206035808' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:10.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:10 smithi086 ceph-mon[24811]: from='client.15672 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:12:10.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:10 smithi086 ceph-mon[24811]: from='client.15678 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:12:10.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:10 smithi086 ceph-mon[24811]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 835 B/s rd, 1 op/s 2024-04-02T05:12:10.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:10 smithi086 ceph-mon[24811]: from='client.25573 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:12:10.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:10 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1206035808' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:10.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:10 smithi086 ceph-mon[29505]: from='client.15672 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:12:10.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:10 smithi086 ceph-mon[29505]: from='client.15678 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:12:10.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:10 smithi086 ceph-mon[29505]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 835 B/s rd, 1 op/s 2024-04-02T05:12:10.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:10 smithi086 ceph-mon[29505]: from='client.25573 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:12:10.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:10 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1206035808' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:12.116 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:11 smithi192 ceph-mon[25697]: from='client.25585 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:12:12.116 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:11 smithi192 ceph-mon[25697]: Upgrade: Updating grafana.a 2024-04-02T05:12:12.116 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:11 smithi192 ceph-mon[25697]: Deploying daemon grafana.a on smithi192 2024-04-02T05:12:12.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:11 smithi086 ceph-mon[24811]: from='client.25585 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:12:12.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:11 smithi086 ceph-mon[24811]: Upgrade: Updating grafana.a 2024-04-02T05:12:12.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:11 smithi086 ceph-mon[24811]: Deploying daemon grafana.a on smithi192 2024-04-02T05:12:12.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:11 smithi086 ceph-mon[29505]: from='client.25585 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:12:12.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:11 smithi086 ceph-mon[29505]: Upgrade: Updating grafana.a 2024-04-02T05:12:12.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:11 smithi086 ceph-mon[29505]: Deploying daemon grafana.a on smithi192 2024-04-02T05:12:12.989 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:12 smithi192 ceph-mon[25697]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 835 B/s rd, 1 op/s 2024-04-02T05:12:13.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:12 smithi086 ceph-mon[24811]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 835 B/s rd, 1 op/s 2024-04-02T05:12:13.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:12 smithi086 ceph-mon[29505]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 835 B/s rd, 1 op/s 2024-04-02T05:12:13.676 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:13 smithi192 systemd[1]: Starting Ceph grafana.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:12:13.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:13 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:12:13] "GET /metrics HTTP/1.1" 200 37983 "" "Prometheus/2.43.0" 2024-04-02T05:12:13.927 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:13 smithi192 podman[66354]: 2024-04-02 05:12:13.673712275 +0000 UTC m=+0.105011339 container create 259af9e63aa540a05375d22d0bca11b8d3b31d70ff20e785000f31710ecad116 (image=quay.io/ceph/grafana:9.4.12, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a, maintainer=Grafana Labs ) 2024-04-02T05:12:13.927 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:13 smithi192 podman[66354]: 2024-04-02 05:12:13.58749373 +0000 UTC m=+0.018792795 image pull 2bacad6d85d802cd176c146528adcc0e47a8a83e94bc4a8b7ba1a9b9ef2c65e7 quay.io/ceph/grafana:9.4.12 2024-04-02T05:12:14.181 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:13 smithi192 podman[66354]: 2024-04-02 05:12:13.959648486 +0000 UTC m=+0.390947567 container init 259af9e63aa540a05375d22d0bca11b8d3b31d70ff20e785000f31710ecad116 (image=quay.io/ceph/grafana:9.4.12, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a, maintainer=Grafana Labs ) 2024-04-02T05:12:14.181 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:13 smithi192 podman[66354]: 2024-04-02 05:12:13.962341419 +0000 UTC m=+0.393640494 container start 259af9e63aa540a05375d22d0bca11b8d3b31d70ff20e785000f31710ecad116 (image=quay.io/ceph/grafana:9.4.12, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a, maintainer=Grafana Labs ) 2024-04-02T05:12:14.181 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 bash[66354]: 259af9e63aa540a05375d22d0bca11b8d3b31d70ff20e785000f31710ecad116 2024-04-02T05:12:14.182 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 systemd[1]: Started Ceph grafana.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:12:14.182 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=settings t=2024-04-02T05:12:14.11037913Z level=info msg="Starting Grafana" version=9.4.12 commit=37ee25ba46 branch=HEAD compiled=2023-05-22T15:58:05Z 2024-04-02T05:12:14.182 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=settings t=2024-04-02T05:12:14.110570176Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-04-02T05:12:14.182 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=settings t=2024-04-02T05:12:14.11058155Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-04-02T05:12:14.182 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=settings t=2024-04-02T05:12:14.110584901Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-04-02T05:12:14.182 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=settings t=2024-04-02T05:12:14.110587924Z level=info msg="Config overridden from command line" arg="default.paths.data=/var/lib/grafana" 2024-04-02T05:12:14.182 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=settings t=2024-04-02T05:12:14.110591191Z level=info msg="Config overridden from command line" arg="default.paths.logs=/var/log/grafana" 2024-04-02T05:12:14.182 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=settings t=2024-04-02T05:12:14.110593662Z level=info msg="Config overridden from command line" arg="default.paths.plugins=/var/lib/grafana/plugins" 2024-04-02T05:12:14.182 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=settings t=2024-04-02T05:12:14.110596254Z level=info msg="Config overridden from command line" arg="default.paths.provisioning=/etc/grafana/provisioning" 2024-04-02T05:12:14.182 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=settings t=2024-04-02T05:12:14.110599021Z level=info msg="Config overridden from command line" arg="default.log.mode=console" 2024-04-02T05:12:14.182 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=settings t=2024-04-02T05:12:14.110601924Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-04-02T05:12:14.182 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=settings t=2024-04-02T05:12:14.11060447Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-04-02T05:12:14.182 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=settings t=2024-04-02T05:12:14.110606896Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-04-02T05:12:14.183 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=settings t=2024-04-02T05:12:14.110610015Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-04-02T05:12:14.183 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=settings t=2024-04-02T05:12:14.11061293Z level=info msg="Path Home" path=/usr/share/grafana 2024-04-02T05:12:14.183 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=settings t=2024-04-02T05:12:14.110615675Z level=info msg="Path Data" path=/var/lib/grafana 2024-04-02T05:12:14.183 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=settings t=2024-04-02T05:12:14.110618074Z level=info msg="Path Logs" path=/var/log/grafana 2024-04-02T05:12:14.183 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=settings t=2024-04-02T05:12:14.110620664Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-04-02T05:12:14.183 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=settings t=2024-04-02T05:12:14.11062324Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-04-02T05:12:14.183 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=settings t=2024-04-02T05:12:14.110626762Z level=info msg="App mode production" 2024-04-02T05:12:14.183 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=sqlstore t=2024-04-02T05:12:14.110674881Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-04-02T05:12:14.183 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=sqlstore t=2024-04-02T05:12:14.110686685Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-04-02T05:12:14.183 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.121747437Z level=info msg="Starting DB migrations" 2024-04-02T05:12:14.183 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.124932264Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-04-02T05:12:14.183 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.136274353Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-04-02T05:12:14.183 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.137410548Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-04-02T05:12:14.183 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.138244771Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-04-02T05:12:14.184 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.139404989Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-04-02T05:12:14.184 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.142128959Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-04-02T05:12:14.184 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.144851816Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-04-02T05:12:14.184 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.147734641Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-04-02T05:12:14.184 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.150330992Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-04-02T05:12:14.184 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.150858942Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-04-02T05:12:14.184 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.153631202Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-04-02T05:12:14.184 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.154758081Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-04-02T05:12:14.185 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.155108868Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-04-02T05:12:14.185 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.156178105Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-04-02T05:12:14.185 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.157240083Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-04-02T05:12:14.185 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.158293316Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-04-02T05:12:14.185 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.158598498Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-04-02T05:12:14.185 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.159700009Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-04-02T05:12:14.185 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.160777263Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-04-02T05:12:14.185 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.161092002Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-04-02T05:12:14.185 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.162158156Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-04-02T05:12:14.185 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.1632445Z level=info msg="Executing migration" id="create provenance_type table" 2024-04-02T05:12:14.185 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.163681669Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-04-02T05:12:14.185 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.166448085Z level=info msg="Executing migration" id="create alert_image table" 2024-04-02T05:12:14.185 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.169158969Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-04-02T05:12:14.185 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.169621322Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-04-02T05:12:14.185 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.169905521Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-04-02T05:12:14.186 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.170336795Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-04-02T05:12:14.186 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.170755943Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-04-02T05:12:14.186 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.170838011Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-04-02T05:12:14.186 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.171131792Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-04-02T05:12:14.186 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.171625635Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-04-02T05:12:14.186 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.172042628Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-04-02T05:12:14.186 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.17234078Z level=info msg="Executing migration" id="create secrets table" 2024-04-02T05:12:14.186 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.172957876Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-04-02T05:12:14.186 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.179179289Z level=info msg="Executing migration" id="add name column into data_keys" 2024-04-02T05:12:14.186 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.180633975Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-04-02T05:12:14.678 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.18099077Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-04-02T05:12:14.678 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.189794381Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-04-02T05:12:14.678 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.196151146Z level=info msg="Executing migration" id="add column hidden to role table" 2024-04-02T05:12:14.678 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.197355019Z level=info msg="Executing migration" id="create query_history table v1" 2024-04-02T05:12:14.678 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.197759818Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-04-02T05:12:14.678 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.198526346Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-04-02T05:12:14.678 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.199304621Z level=info msg="Executing migration" id="teams permissions migration" 2024-04-02T05:12:14.678 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.200086841Z level=info msg="Executing migration" id="dashboard permissions" 2024-04-02T05:12:14.678 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.201386396Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-04-02T05:12:14.678 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.20219458Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-04-02T05:12:14.679 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.202971254Z level=info msg="Executing migration" id="alerting notification permissions" 2024-04-02T05:12:14.679 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.203777119Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-04-02T05:12:14.679 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.204164018Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-04-02T05:12:14.679 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.204985451Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-04-02T05:12:14.679 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.206075385Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-04-02T05:12:14.679 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.206888854Z level=info msg="Executing migration" id="create correlation table v1" 2024-04-02T05:12:14.679 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.207317725Z level=info msg="Executing migration" id="add index correlations.uid" 2024-04-02T05:12:14.679 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.208115374Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-04-02T05:12:14.679 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.208933137Z level=info msg="Executing migration" id="add correlation config column" 2024-04-02T05:12:14.679 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.210039696Z level=info msg="Executing migration" id="create entity_events table" 2024-04-02T05:12:14.679 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.210784699Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-04-02T05:12:14.679 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.211624062Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-04-02T05:12:14.679 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.211701879Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-04-02T05:12:14.679 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.212375496Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-02T05:12:14.680 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.212450886Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-02T05:12:14.680 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.213187667Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-04-02T05:12:14.680 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.213595387Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-04-02T05:12:14.680 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.214379152Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-04-02T05:12:14.680 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.214866482Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-02T05:12:14.680 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.215305361Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-04-02T05:12:14.680 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.217866262Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-04-02T05:12:14.680 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.218304133Z level=info msg="Executing migration" id="Drop public config table" 2024-04-02T05:12:14.680 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.221151439Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-04-02T05:12:14.680 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.223843808Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-04-02T05:12:14.680 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.226498642Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-04-02T05:12:14.680 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.226933036Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-04-02T05:12:14.681 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.229717554Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-04-02T05:12:14.681 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.233668754Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-04-02T05:12:14.681 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.236487503Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-04-02T05:12:14.681 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.237692558Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-04-02T05:12:14.681 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.240353344Z level=info msg="Executing migration" id="add share column" 2024-04-02T05:12:14.681 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.2429864Z level=info msg="Executing migration" id="create default alerting folders" 2024-04-02T05:12:14.681 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.24567405Z level=info msg="Executing migration" id="create file table" 2024-04-02T05:12:14.681 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.246125318Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-04-02T05:12:14.682 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.248782431Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-04-02T05:12:14.682 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.251264029Z level=info msg="Executing migration" id="create file_meta table" 2024-04-02T05:12:14.682 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.253840474Z level=info msg="Executing migration" id="file table idx: path key" 2024-04-02T05:12:14.682 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.254287506Z level=info msg="Executing migration" id="set path collation in file table" 2024-04-02T05:12:14.682 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.256774171Z level=info msg="Executing migration" id="managed permissions migration" 2024-04-02T05:12:14.682 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.2591981Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-04-02T05:12:14.682 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.261675876Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-04-02T05:12:14.682 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.262340196Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-04-02T05:12:14.682 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.264972835Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-04-02T05:12:14.682 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.26741199Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-04-02T05:12:14.682 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.269888018Z level=info msg="Executing migration" id="update group index for alert rules" 2024-04-02T05:12:14.682 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.270319253Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-04-02T05:12:14.682 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.272772381Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-04-02T05:12:14.682 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.273183486Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-04-02T05:12:14.683 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.275300534Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-04-02T05:12:14.683 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.277778305Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-04-02T05:12:14.683 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.278254324Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-04-02T05:12:14.683 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.291470614Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-04-02T05:12:14.683 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.29410835Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-04-02T05:12:14.683 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.296524321Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-04-02T05:12:14.683 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.300955875Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-04-02T05:12:14.683 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.301707052Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-04-02T05:12:14.683 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.302473001Z level=info msg="Executing migration" id="create folder table" 2024-04-02T05:12:14.683 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.302870317Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-04-02T05:12:14.683 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.303348186Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-04-02T05:12:14.683 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.303784114Z level=info msg="Executing migration" id="Update folder title length" 2024-04-02T05:12:14.683 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.304562882Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-04-02T05:12:14.683 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=migrator t=2024-04-02T05:12:14.305017494Z level=info msg="migrations completed" performed=105 skipped=376 duration=180.115348ms 2024-04-02T05:12:14.683 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=sqlstore t=2024-04-02T05:12:14.305324988Z level=info msg="Created default organization" 2024-04-02T05:12:14.684 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=plugin.finder t=2024-04-02T05:12:14.337917791Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled 2024-04-02T05:12:14.684 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=secrets t=2024-04-02T05:12:14.338066167Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-04-02T05:12:14.684 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=query_data t=2024-04-02T05:12:14.340131422Z level=info msg="Query Service initialization" 2024-04-02T05:12:14.684 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=live.push_http t=2024-04-02T05:12:14.344479857Z level=info msg="Live Push Gateway initialization" 2024-04-02T05:12:15.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:14 smithi086 ceph-mon[24811]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 835 B/s rd, 1 op/s 2024-04-02T05:12:15.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:14 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:15.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:14 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:15.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:14 smithi086 ceph-mon[29505]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 835 B/s rd, 1 op/s 2024-04-02T05:12:15.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:14 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:15.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:14 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:15.360 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-mon[25697]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 835 B/s rd, 1 op/s 2024-04-02T05:12:15.361 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:15.361 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:14 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:15.361 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:15 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=infra.usagestats.collector t=2024-04-02T05:12:15.035106943Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-04-02T05:12:15.361 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:15 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=provisioning.datasources t=2024-04-02T05:12:15.037074153Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2024-04-02T05:12:15.361 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:15 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=provisioning.datasources t=2024-04-02T05:12:15.037083481Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-04-02T05:12:15.361 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:15 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=provisioning.datasources t=2024-04-02T05:12:15.037182486Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-04-02T05:12:15.362 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:15 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=provisioning.datasources t=2024-04-02T05:12:15.048299168Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-04-02T05:12:15.362 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:15 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=provisioning.alerting t=2024-04-02T05:12:15.058954191Z level=info msg="starting to provision alerting" 2024-04-02T05:12:15.362 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:15 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=provisioning.alerting t=2024-04-02T05:12:15.058964724Z level=info msg="finished to provision alerting" 2024-04-02T05:12:15.362 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:15 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=ngalert.state.manager t=2024-04-02T05:12:15.059177187Z level=info msg="Warming state cache for startup" 2024-04-02T05:12:15.362 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:15 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=grafanaStorageLogger t=2024-04-02T05:12:15.059200941Z level=info msg="storage starting" 2024-04-02T05:12:15.362 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:15 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=http.server t=2024-04-02T05:12:15.061019478Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-04-02T05:12:15.362 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:15 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=ngalert.state.manager t=2024-04-02T05:12:15.064368813Z level=info msg="State cache has been initialized" states=0 duration=5.190239ms 2024-04-02T05:12:15.362 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:15 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=ticker t=2024-04-02T05:12:15.064431831Z level=info msg=starting first_tick=2024-04-02T05:12:20Z 2024-04-02T05:12:15.362 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:15 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=ngalert.multiorg.alertmanager t=2024-04-02T05:12:15.064419639Z level=info msg="starting MultiOrg Alertmanager" 2024-04-02T05:12:15.363 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:15 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=sqlstore.transactions t=2024-04-02T05:12:15.093705112Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-02T05:12:15.363 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:12:15 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=sqlstore.transactions t=2024-04-02T05:12:15.104761088Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-04-02T05:12:17.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:16 smithi192 ceph-mon[25697]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-02T05:12:17.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:16 smithi086 ceph-mon[24811]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-02T05:12:17.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:16 smithi086 ceph-mon[29505]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-02T05:12:18.254 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:17 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:12:18.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:17 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:12:18.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:17 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:12:19.093 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:18 smithi086 ceph-mon[24811]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-02T05:12:19.093 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:18 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:19.093 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:18 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:19.093 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:18 smithi086 ceph-mon[29505]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-02T05:12:19.093 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:18 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:19.093 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:18 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:19.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:18 smithi192 ceph-mon[25697]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-02T05:12:19.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:18 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:19.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:18 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:20.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:20 smithi192 ceph-mon[25697]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-02T05:12:20.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:20 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:20.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:20 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:20.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:20 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/2738916905' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T05:12:21.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:20 smithi086 ceph-mon[24811]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-02T05:12:21.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:20 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:21.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:20 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:21.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:20 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/2738916905' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T05:12:21.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:20 smithi086 ceph-mon[29505]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-02T05:12:21.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:20 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:21.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:20 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:21.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:20 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/2738916905' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T05:12:22.995 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:22 smithi192 ceph-mon[25697]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:12:23.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:22 smithi086 ceph-mon[24811]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:12:23.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:22 smithi086 ceph-mon[29505]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:12:23.760 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:12:23] "GET /metrics HTTP/1.1" 200 37985 "" "Prometheus/2.43.0" 2024-04-02T05:12:24.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:24 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:24.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:24 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:24.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:24 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:12:24.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:24 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:12:24.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:24 smithi086 ceph-mon[29505]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:12:24.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:24 smithi086 ceph-mon[29505]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:12:24.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:24 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:24.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:24 smithi086 ceph-mon[29505]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T05:12:24.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:24 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:12:24.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:24 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:12:24.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:24 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:12:24.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:24 smithi086 ceph-mon[29505]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T05:12:24.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:24 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:24.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:24 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:24.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:24 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:12:24.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:24 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:12:24.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:24 smithi086 ceph-mon[24811]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:12:24.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:24 smithi086 ceph-mon[24811]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:12:24.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:24 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:24.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:24 smithi086 ceph-mon[24811]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T05:12:24.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:24 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:12:24.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:24 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:12:24.336 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:24 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:12:24.336 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:24 smithi086 ceph-mon[24811]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T05:12:24.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:24 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:24.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:24 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:24.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:24 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:12:24.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:24 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:12:24.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:24 smithi192 ceph-mon[25697]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:12:24.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:24 smithi192 ceph-mon[25697]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:12:24.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:24 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:24.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:24 smithi192 ceph-mon[25697]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T05:12:24.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:24 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:12:24.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:24 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:12:24.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:24 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:12:24.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:24 smithi192 ceph-mon[25697]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T05:12:25.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:25 smithi086 ceph-mon[24811]: Health check update: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-02T05:12:25.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:25 smithi086 ceph-mon[29505]: Health check update: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-02T05:12:25.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:25 smithi192 ceph-mon[25697]: Health check update: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-02T05:12:26.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:12:26.512+0000 7f180812e700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:12:26.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Creating ceph-iscsi config... 2024-04-02T05:12:26.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:12:26.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:12:26.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:12:26.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:12:26.834 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:12:26.834 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Traceback (most recent call last): 2024-04-02T05:12:26.834 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:12:26.834 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: return _run_code(code, main_globals, None, 2024-04-02T05:12:26.834 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:12:26.834 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: exec(code, run_globals) 2024-04-02T05:12:26.834 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:12:26.834 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:12:26.834 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:12:26.834 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:12:26.834 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:12:26.834 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:12:26.834 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:12:26.835 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:12:26.835 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:12:26.835 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Traceback (most recent call last): 2024-04-02T05:12:26.835 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T05:12:26.835 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T05:12:26.835 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T05:12:26.835 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T05:12:26.835 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T05:12:26.836 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: return self.event_loop.get_result(coro, timeout) 2024-04-02T05:12:26.836 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T05:12:26.836 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: return future.result(timeout) 2024-04-02T05:12:26.836 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T05:12:26.836 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: return self.__get_result() 2024-04-02T05:12:26.836 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T05:12:26.836 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: raise self._exception 2024-04-02T05:12:26.836 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T05:12:26.836 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: config_blobs=daemon_spec.final_config, 2024-04-02T05:12:26.836 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T05:12:26.836 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T05:12:26.836 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:12:26.836 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Creating ceph-iscsi config... 2024-04-02T05:12:26.836 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:12:26.836 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:12:26.837 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:12:26.837 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:12:26.837 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:12:26.837 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: Traceback (most recent call last): 2024-04-02T05:12:26.837 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:12:26.837 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: return _run_code(code, main_globals, None, 2024-04-02T05:12:26.837 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:12:26.837 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: exec(code, run_globals) 2024-04-02T05:12:26.837 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:12:26.837 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:12:26.837 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:12:26.837 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:12:26.837 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:12:26.838 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:12:26.838 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:12:26.838 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:12:26.838 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:12:27.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 423 B/s rd, 0 op/s 2024-04-02T05:12:27.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T05:12:27.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:12:27.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:27.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:12:27.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:12:27.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-02T05:12:27.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-02T05:12:27.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:12:27.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:12:27.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-02T05:12:27.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 423 B/s rd, 0 op/s 2024-04-02T05:12:27.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T05:12:27.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:12:27.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.336 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.336 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.336 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.336 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.336 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.336 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.336 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.336 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.336 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.336 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.336 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.336 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.336 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.336 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:27.337 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:12:27.337 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:12:27.337 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-02T05:12:27.337 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.337 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.337 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-02T05:12:27.337 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:12:27.337 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:12:27.337 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-02T05:12:27.338 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:12:27.338 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:12:27.338 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:12:27.338 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:12:27.338 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-02T05:12:27.338 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:12:27.338 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:12:27.338 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:12:27.338 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:12:27.338 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-02T05:12:27.339 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.339 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.339 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:12:27.339 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:12:27.339 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-02T05:12:27.339 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:12:27.339 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:12:27.339 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:12:27.339 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:12:27.339 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-02T05:12:27.339 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:12:27.339 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:12:27.339 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-02T05:12:27.339 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.340 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.340 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.340 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.340 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.340 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.340 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.340 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.340 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.340 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.340 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.340 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.340 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:12:27.340 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:12:27.340 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-02T05:12:27.340 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:12:27.341 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:12:27.341 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:27.341 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:12:27.341 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:12:27.341 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:12:27.341 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:12:27.341 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:12:27.341 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:12:27.341 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:12:27.341 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-02T05:12:27.342 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:12:27.342 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:12:27.342 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:12:27.342 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:12:27.342 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-02T05:12:27.342 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.342 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.342 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:12:27.342 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:12:27.342 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-02T05:12:27.342 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:12:27.342 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:12:27.342 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:12:27.342 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:12:27.343 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-02T05:12:27.343 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:12:27.343 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:12:27.343 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-02T05:12:27.343 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.343 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.343 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.343 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.343 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.343 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.343 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.343 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.343 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.343 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.343 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.344 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.344 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:12:27.344 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:12:27.344 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-02T05:12:27.344 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:12:27.344 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:12:27.344 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:27.344 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:12:27.344 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:12:27.344 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:26 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:12:27.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 423 B/s rd, 0 op/s 2024-04-02T05:12:27.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T05:12:27.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:12:27.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:27.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:27.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:12:27.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:12:27.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-02T05:12:27.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-02T05:12:27.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:12:27.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:12:27.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-02T05:12:27.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:12:27.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:12:27.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:12:27.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:12:27.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-02T05:12:27.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:12:27.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:12:27.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:12:27.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:12:27.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-02T05:12:27.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:12:27.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:12:27.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-02T05:12:27.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:12:27.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:12:27.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:12:27.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:12:27.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-02T05:12:27.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:12:27.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:12:27.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-02T05:12:27.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:12:27.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:12:27.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:12:27.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-02T05:12:27.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:12:27.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:12:27.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:27.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:12:27.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:12:27.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:26 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:12:28.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:27 smithi086 ceph-mon[24811]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:12:28.084 INFO:journalctl@ceph.mon.a.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T05:12:28.084 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:12:28.084 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:12:28.084 INFO:journalctl@ceph.mon.a.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:12:28.084 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:12:28.084 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:12:28.084 INFO:journalctl@ceph.mon.a.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:12:28.084 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:12:28.084 INFO:journalctl@ceph.mon.a.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:12:28.084 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:12:28.084 INFO:journalctl@ceph.mon.a.smithi086.stdout: exec(code, run_globals) 2024-04-02T05:12:28.084 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:12:28.084 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:12:28.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:12:28.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:12:28.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:12:28.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:12:28.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:12:28.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:12:28.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:12:28.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:12:28.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T05:12:28.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T05:12:28.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T05:12:28.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T05:12:28.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T05:12:28.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-02T05:12:28.085 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T05:12:28.086 INFO:journalctl@ceph.mon.a.smithi086.stdout: return future.result(timeout) 2024-04-02T05:12:28.086 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T05:12:28.086 INFO:journalctl@ceph.mon.a.smithi086.stdout: return self.__get_result() 2024-04-02T05:12:28.086 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T05:12:28.086 INFO:journalctl@ceph.mon.a.smithi086.stdout: raise self._exception 2024-04-02T05:12:28.086 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T05:12:28.086 INFO:journalctl@ceph.mon.a.smithi086.stdout: config_blobs=daemon_spec.final_config, 2024-04-02T05:12:28.086 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T05:12:28.086 INFO:journalctl@ceph.mon.a.smithi086.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T05:12:28.086 INFO:journalctl@ceph.mon.a.smithi086.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:12:28.086 INFO:journalctl@ceph.mon.a.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T05:12:28.086 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:12:28.086 INFO:journalctl@ceph.mon.a.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:12:28.086 INFO:journalctl@ceph.mon.a.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:12:28.087 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:12:28.087 INFO:journalctl@ceph.mon.a.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:12:28.087 INFO:journalctl@ceph.mon.a.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:12:28.087 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:12:28.087 INFO:journalctl@ceph.mon.a.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:12:28.087 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:12:28.087 INFO:journalctl@ceph.mon.a.smithi086.stdout: exec(code, run_globals) 2024-04-02T05:12:28.087 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:12:28.087 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:12:28.087 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:12:28.087 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:12:28.088 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:27 smithi086 ceph-mon[29505]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:12:28.088 INFO:journalctl@ceph.mon.c.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T05:12:28.088 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:12:28.088 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:12:28.088 INFO:journalctl@ceph.mon.c.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:12:28.088 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:12:28.088 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:12:28.088 INFO:journalctl@ceph.mon.c.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:12:28.088 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:12:28.088 INFO:journalctl@ceph.mon.c.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:12:28.088 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:12:28.088 INFO:journalctl@ceph.mon.c.smithi086.stdout: exec(code, run_globals) 2024-04-02T05:12:28.088 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:12:28.089 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:12:28.089 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:12:28.089 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:12:28.089 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:12:28.089 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:12:28.089 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:12:28.089 INFO:journalctl@ceph.mon.c.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:12:28.089 INFO:journalctl@ceph.mon.c.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:12:28.089 INFO:journalctl@ceph.mon.c.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:12:28.089 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T05:12:28.089 INFO:journalctl@ceph.mon.c.smithi086.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T05:12:28.089 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T05:12:28.089 INFO:journalctl@ceph.mon.c.smithi086.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T05:12:28.089 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T05:12:28.090 INFO:journalctl@ceph.mon.c.smithi086.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-02T05:12:28.090 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T05:12:28.090 INFO:journalctl@ceph.mon.c.smithi086.stdout: return future.result(timeout) 2024-04-02T05:12:28.090 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T05:12:28.090 INFO:journalctl@ceph.mon.c.smithi086.stdout: return self.__get_result() 2024-04-02T05:12:28.090 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T05:12:28.090 INFO:journalctl@ceph.mon.c.smithi086.stdout: raise self._exception 2024-04-02T05:12:28.090 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T05:12:28.090 INFO:journalctl@ceph.mon.c.smithi086.stdout: config_blobs=daemon_spec.final_config, 2024-04-02T05:12:28.090 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T05:12:28.090 INFO:journalctl@ceph.mon.c.smithi086.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T05:12:28.090 INFO:journalctl@ceph.mon.c.smithi086.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:12:28.090 INFO:journalctl@ceph.mon.c.smithi086.stdout: Creating ceph-iscsi config... 2024-04-02T05:12:28.090 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:12:28.090 INFO:journalctl@ceph.mon.c.smithi086.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:12:28.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:12:28.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:12:28.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:12:28.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: Traceback (most recent call last): 2024-04-02T05:12:28.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:12:28.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:12:28.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:12:28.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: exec(code, run_globals) 2024-04-02T05:12:28.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:12:28.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:12:28.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:12:28.091 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:12:28.092 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:12:28.092 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:12:28.092 INFO:journalctl@ceph.mon.a.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:12:28.092 INFO:journalctl@ceph.mon.a.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:12:28.092 INFO:journalctl@ceph.mon.a.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:12:28.092 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:27 smithi086 ceph-mon[24811]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T05:12:28.092 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:27 smithi086 ceph-mon[24811]: Upgrade: Finalizing container_image settings 2024-04-02T05:12:28.092 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:27 smithi086 ceph-mon[24811]: Upgrade: Complete! 2024-04-02T05:12:28.092 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:27 smithi086 ceph-mon[24811]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 442 B/s rd, 0 op/s 2024-04-02T05:12:28.092 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:27 smithi086 ceph-mon[24811]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 563 B/s rd, 0 op/s 2024-04-02T05:12:28.092 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:27 smithi086 ceph-mon[24811]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T05:12:28.092 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:27 smithi086 ceph-mon[24811]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T05:12:28.093 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:27 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:28.093 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:12:28.093 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:12:28.093 INFO:journalctl@ceph.mon.c.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:12:28.093 INFO:journalctl@ceph.mon.c.smithi086.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:12:28.093 INFO:journalctl@ceph.mon.c.smithi086.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:12:28.093 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:27 smithi086 ceph-mon[29505]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T05:12:28.093 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:27 smithi086 ceph-mon[29505]: Upgrade: Finalizing container_image settings 2024-04-02T05:12:28.093 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:27 smithi086 ceph-mon[29505]: Upgrade: Complete! 2024-04-02T05:12:28.093 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:27 smithi086 ceph-mon[29505]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 442 B/s rd, 0 op/s 2024-04-02T05:12:28.093 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:27 smithi086 ceph-mon[29505]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 563 B/s rd, 0 op/s 2024-04-02T05:12:28.094 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:27 smithi086 ceph-mon[29505]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T05:12:28.094 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:27 smithi086 ceph-mon[29505]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T05:12:28.094 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:27 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:28.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:27 smithi192 ceph-mon[25697]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:12:28.428 INFO:journalctl@ceph.mon.b.smithi192.stdout: Creating ceph-iscsi config... 2024-04-02T05:12:28.428 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:12:28.428 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:12:28.428 INFO:journalctl@ceph.mon.b.smithi192.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:12:28.428 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:12:28.428 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:12:28.428 INFO:journalctl@ceph.mon.b.smithi192.stdout: Traceback (most recent call last): 2024-04-02T05:12:28.428 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:12:28.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:12:28.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:12:28.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: exec(code, run_globals) 2024-04-02T05:12:28.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:12:28.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:12:28.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:12:28.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:12:28.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:12:28.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:12:28.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:12:28.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:12:28.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:12:28.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: Traceback (most recent call last): 2024-04-02T05:12:28.429 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-02T05:12:28.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-02T05:12:28.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-02T05:12:28.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-02T05:12:28.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-02T05:12:28.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-02T05:12:28.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-02T05:12:28.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: return future.result(timeout) 2024-04-02T05:12:28.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-02T05:12:28.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: return self.__get_result() 2024-04-02T05:12:28.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-02T05:12:28.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: raise self._exception 2024-04-02T05:12:28.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-02T05:12:28.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: config_blobs=daemon_spec.final_config, 2024-04-02T05:12:28.430 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-02T05:12:28.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-02T05:12:28.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:12:28.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: Creating ceph-iscsi config... 2024-04-02T05:12:28.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:12:28.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:12:28.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: Non-zero exit code 1 from systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:12:28.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:12:28.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:12:28.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: Traceback (most recent call last): 2024-04-02T05:12:28.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:12:28.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:12:28.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:12:28.431 INFO:journalctl@ceph.mon.b.smithi192.stdout: exec(code, run_globals) 2024-04-02T05:12:28.432 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:12:28.432 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:12:28.432 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:12:28.432 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:12:28.432 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:12:28.432 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1067, in deploy_daemon 2024-04-02T05:12:28.432 INFO:journalctl@ceph.mon.b.smithi192.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:12:28.432 INFO:journalctl@ceph.mon.b.smithi192.stdout: RuntimeError: Failed command: systemctl restart ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:12:28.432 INFO:journalctl@ceph.mon.b.smithi192.stdout: See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:12:28.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:27 smithi192 ceph-mon[25697]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T05:12:28.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:27 smithi192 ceph-mon[25697]: Upgrade: Finalizing container_image settings 2024-04-02T05:12:28.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:27 smithi192 ceph-mon[25697]: Upgrade: Complete! 2024-04-02T05:12:28.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:27 smithi192 ceph-mon[25697]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 442 B/s rd, 0 op/s 2024-04-02T05:12:28.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:27 smithi192 ceph-mon[25697]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 563 B/s rd, 0 op/s 2024-04-02T05:12:28.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:27 smithi192 ceph-mon[25697]: Reconfiguring iscsi.foo.smithi086.rsbqwx (dependencies changed)... 2024-04-02T05:12:28.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:27 smithi192 ceph-mon[25697]: Reconfiguring daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T05:12:28.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:27 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:30.229 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:29 smithi192 ceph-mon[25697]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 563 B/s rd, 0 op/s 2024-04-02T05:12:30.229 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:29 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:30.229 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:29 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:30.229 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:29 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T05:12:30.229 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:29 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T05:12:30.324 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:29 smithi086 ceph-mon[29505]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 563 B/s rd, 0 op/s 2024-04-02T05:12:30.324 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:29 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:30.324 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:29 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:30.324 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:29 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T05:12:30.324 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:29 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T05:12:30.324 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:29 smithi086 ceph-mon[24811]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 563 B/s rd, 0 op/s 2024-04-02T05:12:30.324 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:29 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:30.325 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:29 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:30.325 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:29 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T05:12:30.325 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:29 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T05:12:31.241 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:30 smithi192 ceph-mon[25697]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T05:12:31.241 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:30 smithi192 ceph-mon[25697]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T05:12:31.241 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:30 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1619700409' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T05:12:31.284 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:30 smithi086 ceph-mon[29505]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T05:12:31.284 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:30 smithi086 ceph-mon[29505]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T05:12:31.284 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:30 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1619700409' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T05:12:31.284 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:30 smithi086 ceph-mon[24811]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T05:12:31.285 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:30 smithi086 ceph-mon[24811]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T05:12:31.285 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:30 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1619700409' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T05:12:32.332 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:31 smithi086 ceph-mon[24811]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 809 B/s rd, 1 op/s 2024-04-02T05:12:32.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:31 smithi086 ceph-mon[29505]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 809 B/s rd, 1 op/s 2024-04-02T05:12:32.381 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:31 smithi192 ceph-mon[25697]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 809 B/s rd, 1 op/s 2024-04-02T05:12:33.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:32 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:12:33.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:32 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:12:33.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:32 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:12:33.668 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:33 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:12:33] "GET /metrics HTTP/1.1" 200 37979 "" "Prometheus/2.43.0" 2024-04-02T05:12:34.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:33 smithi086 ceph-mon[24811]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 640 B/s rd, 0 op/s 2024-04-02T05:12:34.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:33 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:34.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:33 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:34.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:33 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:12:34.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:33 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:12:34.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:33 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:34.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:33 smithi086 ceph-mon[29505]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 640 B/s rd, 0 op/s 2024-04-02T05:12:34.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:33 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:34.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:33 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:34.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:33 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:12:34.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:33 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:12:34.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:33 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:34.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:33 smithi192 ceph-mon[25697]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 640 B/s rd, 0 op/s 2024-04-02T05:12:34.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:33 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:34.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:33 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:34.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:33 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:12:34.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:33 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:12:34.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:33 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:35.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:34 smithi086 ceph-mon[24811]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 806 B/s rd, 1 op/s 2024-04-02T05:12:35.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:34 smithi086 ceph-mon[24811]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-02T05:12:35.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:34 smithi086 ceph-mon[24811]: Cluster is now healthy 2024-04-02T05:12:35.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:34 smithi086 ceph-mon[29505]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 806 B/s rd, 1 op/s 2024-04-02T05:12:35.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:34 smithi086 ceph-mon[29505]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-02T05:12:35.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:34 smithi086 ceph-mon[29505]: Cluster is now healthy 2024-04-02T05:12:35.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:34 smithi192 ceph-mon[25697]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 806 B/s rd, 1 op/s 2024-04-02T05:12:35.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:34 smithi192 ceph-mon[25697]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-02T05:12:35.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:34 smithi192 ceph-mon[25697]: Cluster is now healthy 2024-04-02T05:12:37.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:36 smithi086 ceph-mon[24811]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 771 B/s rd, 0 op/s 2024-04-02T05:12:37.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:36 smithi086 ceph-mon[29505]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 771 B/s rd, 0 op/s 2024-04-02T05:12:37.434 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:36 smithi192 ceph-mon[25697]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 771 B/s rd, 0 op/s 2024-04-02T05:12:39.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:38 smithi086 ceph-mon[24811]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 634 B/s rd, 0 op/s 2024-04-02T05:12:39.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:38 smithi086 ceph-mon[29505]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 634 B/s rd, 0 op/s 2024-04-02T05:12:39.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:38 smithi192 ceph-mon[25697]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 634 B/s rd, 0 op/s 2024-04-02T05:12:41.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:40 smithi086 ceph-mon[29505]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:12:41.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:40 smithi086 ceph-mon[29505]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:12:41.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:40 smithi086 ceph-mon[24811]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:12:41.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:40 smithi086 ceph-mon[24811]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:12:41.421 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-04-02T05:12:41.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:40 smithi192 ceph-mon[25697]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:12:41.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:40 smithi192 ceph-mon[25697]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:12:42.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:41 smithi192 ceph-mon[25697]: from='client.15750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:12:42.332 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:41 smithi086 ceph-mon[24811]: from='client.15750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:12:42.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:41 smithi086 ceph-mon[29505]: from='client.15750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:12:43.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:42 smithi086 ceph-mon[24811]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 906 B/s rd, 0 op/s 2024-04-02T05:12:43.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:42 smithi086 ceph-mon[29505]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 906 B/s rd, 0 op/s 2024-04-02T05:12:43.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:42 smithi192 ceph-mon[25697]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 906 B/s rd, 0 op/s 2024-04-02T05:12:43.449 INFO:teuthology.orchestra.run.smithi086.stdout:true 2024-04-02T05:12:43.753 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:43 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:12:43] "GET /metrics HTTP/1.1" 200 37979 "" "Prometheus/2.43.0" 2024-04-02T05:12:44.070 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:43 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/2359998631' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:44.070 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:43 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/2359998631' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:44.094 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph versions | jq -e '"'"'.mgr | keys'"'"' | grep $sha1' 2024-04-02T05:12:44.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:43 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/2359998631' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:45.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:44 smithi086 ceph-mon[24811]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-02T05:12:45.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:44 smithi086 ceph-mon[29505]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-02T05:12:45.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:44 smithi192 ceph-mon[25697]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-02T05:12:46.319 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)" 2024-04-02T05:12:46.940 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 2'"'"'' 2024-04-02T05:12:47.246 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:46 smithi086 ceph-mon[29505]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:12:47.246 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:46 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/3321023373' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:47.247 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:46 smithi086 ceph-mon[24811]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:12:47.247 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:46 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/3321023373' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:47.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:46 smithi192 ceph-mon[25697]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:12:47.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:46 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/3321023373' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:48.997 INFO:teuthology.orchestra.run.smithi086.stdout:true 2024-04-02T05:12:49.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:48 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:49.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:48 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:12:49.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:48 smithi086 ceph-mon[24811]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:12:49.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:48 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:49.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:48 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:12:49.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:48 smithi086 ceph-mon[29505]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:12:49.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:48 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:12:49.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:48 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:12:49.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:48 smithi192 ceph-mon[25697]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:12:49.701 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 2'"'"'' 2024-04-02T05:12:50.008 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:49 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/2407793343' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:50.008 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:49 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/2407793343' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:50.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:49 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/2407793343' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:12:51.064 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:50 smithi086 ceph-mon[29505]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:12:51.064 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:50 smithi086 ceph-mon[29505]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:12:51.064 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:50 smithi086 ceph-mon[24811]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:12:51.064 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:50 smithi086 ceph-mon[24811]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:12:51.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:50 smithi192 ceph-mon[25697]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:12:51.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:50 smithi192 ceph-mon[25697]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:12:53.211 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:52 smithi086 ceph-mon[24811]: from='client.25663 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:12:53.211 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:52 smithi086 ceph-mon[24811]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:12:53.211 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:52 smithi086 ceph-mon[29505]: from='client.25663 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:12:53.211 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:52 smithi086 ceph-mon[29505]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:12:53.283 INFO:teuthology.orchestra.run.smithi086.stdout:true 2024-04-02T05:12:53.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:52 smithi192 ceph-mon[25697]: from='client.25663 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:12:53.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:52 smithi192 ceph-mon[25697]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:12:53.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:12:53 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:12:53] "GET /metrics HTTP/1.1" 200 37991 "" "Prometheus/2.43.0" 2024-04-02T05:12:54.032 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch upgrade status' 2024-04-02T05:12:55.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:54 smithi086 ceph-mon[24811]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:12:55.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:54 smithi086 ceph-mon[29505]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:12:55.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:54 smithi192 ceph-mon[25697]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:12:56.125 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:12:56.125 INFO:teuthology.orchestra.run.smithi086.stdout: "target_image": null, 2024-04-02T05:12:56.125 INFO:teuthology.orchestra.run.smithi086.stdout: "in_progress": false, 2024-04-02T05:12:56.125 INFO:teuthology.orchestra.run.smithi086.stdout: "which": "", 2024-04-02T05:12:56.125 INFO:teuthology.orchestra.run.smithi086.stdout: "services_complete": [], 2024-04-02T05:12:56.125 INFO:teuthology.orchestra.run.smithi086.stdout: "progress": null, 2024-04-02T05:12:56.125 INFO:teuthology.orchestra.run.smithi086.stdout: "message": "", 2024-04-02T05:12:56.126 INFO:teuthology.orchestra.run.smithi086.stdout: "is_paused": false 2024-04-02T05:12:56.126 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:12:56.815 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph health detail' 2024-04-02T05:12:57.082 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:56 smithi086 ceph-mon[24811]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:12:57.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:56 smithi086 ceph-mon[29505]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:12:57.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:56 smithi192 ceph-mon[25697]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:12:58.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:57 smithi086 ceph-mon[24811]: from='client.15780 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:12:58.087 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:57 smithi086 ceph-mon[29505]: from='client.15780 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:12:58.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:57 smithi192 ceph-mon[25697]: from='client.15780 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:12:58.886 INFO:teuthology.orchestra.run.smithi086.stdout:HEALTH_OK 2024-04-02T05:12:59.274 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:58 smithi086 ceph-mon[24811]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:12:59.274 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:12:58 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1952997583' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T05:12:59.274 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:58 smithi086 ceph-mon[29505]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:12:59.275 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:12:58 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1952997583' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T05:12:59.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:58 smithi192 ceph-mon[25697]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:12:59.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:12:58 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1952997583' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T05:12:59.583 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '"'"'{print $2}'"'"')' 2024-04-02T05:13:01.143 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:00 smithi086 ceph-mon[29505]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:13:01.143 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:00 smithi086 ceph-mon[29505]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:13:01.143 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:00 smithi086 ceph-mon[24811]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:13:01.143 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:00 smithi086 ceph-mon[24811]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:13:01.229 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:00 smithi192 ceph-mon[25697]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:13:01.229 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:00 smithi192 ceph-mon[25697]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:13:03.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:02 smithi086 ceph-mon[24811]: from='client.25681 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:13:03.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:02 smithi086 ceph-mon[24811]: from='client.25687 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "daemon_types": "mon", "hosts": "smithi192", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:13:03.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:02 smithi086 ceph-mon[24811]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:03.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:02 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:13:03.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:02 smithi086 ceph-mon[29505]: from='client.25681 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:13:03.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:02 smithi086 ceph-mon[29505]: from='client.25687 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "daemon_types": "mon", "hosts": "smithi192", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:13:03.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:02 smithi086 ceph-mon[29505]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:03.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:02 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:13:03.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:02 smithi192 ceph-mon[25697]: from='client.25681 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:13:03.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:02 smithi192 ceph-mon[25697]: from='client.25687 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "daemon_types": "mon", "hosts": "smithi192", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:13:03.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:02 smithi192 ceph-mon[25697]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:03.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:02 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:13:03.542 INFO:teuthology.orchestra.run.smithi086.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:13:03.832 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:13:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:13:03] "GET /metrics HTTP/1.1" 200 37991 "" "Prometheus/2.43.0" 2024-04-02T05:13:04.201 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-02T05:13:04.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:04 smithi086 ceph-mon[29505]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:13:04.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:04 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:04.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:04 smithi086 ceph-mon[29505]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:13:04.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:04 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:13:04.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:04 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:13:04.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:04 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:04.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:04 smithi086 ceph-mon[24811]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:13:04.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:04 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:04.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:04 smithi086 ceph-mon[24811]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:13:04.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:04 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:13:04.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:04 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:13:04.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:04 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:04.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:04 smithi192 ceph-mon[25697]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:13:04.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:04 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:04.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:04 smithi192 ceph-mon[25697]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:13:04.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:04 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:13:04.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:04 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:13:04.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:04 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:05.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:05 smithi086 ceph-mon[24811]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:13:05.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:05 smithi086 ceph-mon[29505]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:13:05.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:05 smithi192 ceph-mon[25697]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:13:06.519 INFO:teuthology.orchestra.run.smithi086.stdout:true 2024-04-02T05:13:07.007 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:06 smithi086 ceph-mon[29505]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:07.007 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:06 smithi086 ceph-mon[24811]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:07.190 INFO:teuthology.orchestra.run.smithi086.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T05:13:07.190 INFO:teuthology.orchestra.run.smithi086.stdout:alertmanager.a smithi086 *:9093,9094 running (71s) 33s ago 23m 16.0M - 0.25.0 c8568f914cd2 660ea62bb6b9 2024-04-02T05:13:07.190 INFO:teuthology.orchestra.run.smithi086.stdout:grafana.a smithi192 *:3000 running (53s) 48s ago 22m 49.2M - 9.4.12 2bacad6d85d8 259af9e63aa5 2024-04-02T05:13:07.190 INFO:teuthology.orchestra.run.smithi086.stdout:iscsi.foo.smithi086.rsbqwx smithi086 running (38s) 33s ago 22m 52.0M - 3.5 e1d6a67b021e fe728eb940f7 2024-04-02T05:13:07.190 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.x smithi192 *:8443,9283,8765 running (2m) 48s ago 27m 449M - 19.0.0-1979-g6cec793d 78f830190839 dd1e6d0f0a95 2024-04-02T05:13:07.190 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.y smithi086 *:8443,9283,8765 running (11m) 33s ago 29m 510M - 19.0.0-1979-g6cec793d 78f830190839 3dca54a018ab 2024-04-02T05:13:07.190 INFO:teuthology.orchestra.run.smithi086.stdout:mon.a smithi086 running (29m) 33s ago 29m 79.8M 2048M 17.2.0 e1d6a67b021e b6c507ab96f8 2024-04-02T05:13:07.190 INFO:teuthology.orchestra.run.smithi086.stdout:mon.b smithi192 running (27m) 48s ago 27m 60.1M 2048M 17.2.0 e1d6a67b021e 4dadd8d1c318 2024-04-02T05:13:07.190 INFO:teuthology.orchestra.run.smithi086.stdout:mon.c smithi086 running (27m) 33s ago 27m 58.3M 2048M 17.2.0 e1d6a67b021e 49275483a2d4 2024-04-02T05:13:07.190 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.a smithi086 *:9100 running (116s) 33s ago 23m 15.8M - 1.5.0 0da6a335fe13 05a844569d7b 2024-04-02T05:13:07.190 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.b smithi192 *:9100 running (109s) 48s ago 23m 17.2M - 1.5.0 0da6a335fe13 1a7d6788d58b 2024-04-02T05:13:07.190 INFO:teuthology.orchestra.run.smithi086.stdout:osd.0 smithi086 running (27m) 33s ago 27m 57.5M 2992M 17.2.0 e1d6a67b021e 6d90c60d1c71 2024-04-02T05:13:07.191 INFO:teuthology.orchestra.run.smithi086.stdout:osd.1 smithi086 running (26m) 33s ago 26m 56.5M 2992M 17.2.0 e1d6a67b021e 203689f71603 2024-04-02T05:13:07.191 INFO:teuthology.orchestra.run.smithi086.stdout:osd.2 smithi086 running (26m) 33s ago 26m 50.4M 2992M 17.2.0 e1d6a67b021e 472c860a9af8 2024-04-02T05:13:07.191 INFO:teuthology.orchestra.run.smithi086.stdout:osd.3 smithi086 running (26m) 33s ago 26m 52.7M 2992M 17.2.0 e1d6a67b021e 327538dac44e 2024-04-02T05:13:07.191 INFO:teuthology.orchestra.run.smithi086.stdout:osd.4 smithi192 running (25m) 48s ago 25m 56.0M 3248M 17.2.0 e1d6a67b021e 6535d7919a2d 2024-04-02T05:13:07.191 INFO:teuthology.orchestra.run.smithi086.stdout:osd.5 smithi192 running (25m) 48s ago 25m 54.8M 3248M 17.2.0 e1d6a67b021e 7259249605f2 2024-04-02T05:13:07.191 INFO:teuthology.orchestra.run.smithi086.stdout:osd.6 smithi192 running (24m) 48s ago 24m 52.8M 3248M 17.2.0 e1d6a67b021e bb7d171b02a1 2024-04-02T05:13:07.191 INFO:teuthology.orchestra.run.smithi086.stdout:osd.7 smithi192 running (24m) 48s ago 24m 54.5M 3248M 17.2.0 e1d6a67b021e 95f5a0353999 2024-04-02T05:13:07.191 INFO:teuthology.orchestra.run.smithi086.stdout:prometheus.a smithi192 *:9095 running (90s) 48s ago 23m 49.0M - 2.43.0 a07b618ecd1d 8378fcdb60e3 2024-04-02T05:13:07.191 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi086.bxzxlp smithi086 *:8000 running (22m) 33s ago 22m 78.0M - 17.2.0 e1d6a67b021e 6725785fbc53 2024-04-02T05:13:07.191 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi192.pmlbot smithi192 *:8000 running (22m) 48s ago 22m 77.2M - 17.2.0 e1d6a67b021e 5ae3638f8b1c 2024-04-02T05:13:07.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:06 smithi192 ceph-mon[25697]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:07.577 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:13:07.578 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": { 2024-04-02T05:13:07.578 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-02T05:13:07.578 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:13:07.578 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": { 2024-04-02T05:13:07.578 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 2 2024-04-02T05:13:07.578 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:13:07.578 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": { 2024-04-02T05:13:07.578 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-02T05:13:07.578 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:13:07.578 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": {}, 2024-04-02T05:13:07.578 INFO:teuthology.orchestra.run.smithi086.stdout: "rgw": { 2024-04-02T05:13:07.579 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-02T05:13:07.579 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:13:07.579 INFO:teuthology.orchestra.run.smithi086.stdout: "overall": { 2024-04-02T05:13:07.579 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-02T05:13:07.579 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 2 2024-04-02T05:13:07.579 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-04-02T05:13:07.579 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:13:07.904 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:13:07.949 INFO:teuthology.orchestra.run.smithi086.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", 2024-04-02T05:13:07.950 INFO:teuthology.orchestra.run.smithi086.stdout: "in_progress": true, 2024-04-02T05:13:07.950 INFO:teuthology.orchestra.run.smithi086.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi192", 2024-04-02T05:13:07.950 INFO:teuthology.orchestra.run.smithi086.stdout: "services_complete": [], 2024-04-02T05:13:07.950 INFO:teuthology.orchestra.run.smithi086.stdout: "progress": "0/1 daemons upgraded", 2024-04-02T05:13:07.950 INFO:teuthology.orchestra.run.smithi086.stdout: "message": "Currently upgrading mon daemons", 2024-04-02T05:13:07.950 INFO:teuthology.orchestra.run.smithi086.stdout: "is_paused": false 2024-04-02T05:13:07.951 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:13:08.242 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:07 smithi192 ceph-mon[25697]: from='client.25693 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:13:08.242 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:07 smithi192 ceph-mon[25697]: from='client.25699 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:13:08.242 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:07 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:08.242 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:07 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:13:08.242 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:07 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:08.242 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:07 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:08.242 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:07 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-02T05:13:08.243 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:07 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-02T05:13:08.243 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:07 smithi192 ceph-mon[25697]: from='client.? 172.21.15.86:0/1646050644' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:08.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:07 smithi086 ceph-mon[24811]: from='client.25693 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:13:08.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:07 smithi086 ceph-mon[24811]: from='client.25699 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:13:08.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:07 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:08.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:07 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:13:08.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:07 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:08.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:07 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:08.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:07 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-02T05:13:08.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:07 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-02T05:13:08.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:07 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1646050644' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:08.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:07 smithi086 ceph-mon[29505]: from='client.25693 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:13:08.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:07 smithi086 ceph-mon[29505]: from='client.25699 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:13:08.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:07 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:08.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:07 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:13:08.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:07 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:08.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:07 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:08.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:07 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-02T05:13:08.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:07 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-02T05:13:08.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:07 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1646050644' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:09.257 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:08 smithi192 ceph-mon[25697]: Upgrade: Target is version 19.0.0-1979-g6cec793d (squid) 2024-04-02T05:13:09.257 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:08 smithi192 ceph-mon[25697]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7bd5b84926d4a0d6c51e5562e965bff211cec2740c9f4dec9b5e00a2bbb75eae'] 2024-04-02T05:13:09.257 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:08 smithi192 ceph-mon[25697]: Upgrade: Setting container_image for all mgr 2024-04-02T05:13:09.257 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:08 smithi192 ceph-mon[25697]: Upgrade: It appears safe to stop mon.b 2024-04-02T05:13:09.257 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:08 smithi192 ceph-mon[25697]: from='client.15816 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:13:09.257 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:08 smithi192 ceph-mon[25697]: from='client.15828 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:13:09.257 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:08 smithi192 ceph-mon[25697]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:09.257 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:08 smithi192 ceph-mon[25697]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:09.258 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:08 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T05:13:09.258 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:08 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T05:13:09.258 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:08 smithi192 ceph-mon[25697]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:13:09.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:08 smithi086 ceph-mon[24811]: Upgrade: Target is version 19.0.0-1979-g6cec793d (squid) 2024-04-02T05:13:09.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:08 smithi086 ceph-mon[24811]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7bd5b84926d4a0d6c51e5562e965bff211cec2740c9f4dec9b5e00a2bbb75eae'] 2024-04-02T05:13:09.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:08 smithi086 ceph-mon[24811]: Upgrade: Setting container_image for all mgr 2024-04-02T05:13:09.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:08 smithi086 ceph-mon[24811]: Upgrade: It appears safe to stop mon.b 2024-04-02T05:13:09.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:08 smithi086 ceph-mon[24811]: from='client.15816 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:13:09.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:08 smithi086 ceph-mon[24811]: from='client.15828 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:13:09.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:08 smithi086 ceph-mon[24811]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:09.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:08 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:09.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:08 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T05:13:09.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:08 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T05:13:09.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:08 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:13:09.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:08 smithi086 ceph-mon[29505]: Upgrade: Target is version 19.0.0-1979-g6cec793d (squid) 2024-04-02T05:13:09.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:08 smithi086 ceph-mon[29505]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7bd5b84926d4a0d6c51e5562e965bff211cec2740c9f4dec9b5e00a2bbb75eae'] 2024-04-02T05:13:09.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:08 smithi086 ceph-mon[29505]: Upgrade: Setting container_image for all mgr 2024-04-02T05:13:09.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:08 smithi086 ceph-mon[29505]: Upgrade: It appears safe to stop mon.b 2024-04-02T05:13:09.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:08 smithi086 ceph-mon[29505]: from='client.15816 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:13:09.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:08 smithi086 ceph-mon[29505]: from='client.15828 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:13:09.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:08 smithi086 ceph-mon[29505]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:09.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:08 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:09.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:08 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T05:13:09.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:08 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T05:13:09.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:08 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:13:10.273 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:09 smithi192 ceph-mon[25697]: Upgrade: Updating mon.b 2024-04-02T05:13:10.274 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:09 smithi192 ceph-mon[25697]: Deploying daemon mon.b on smithi192 2024-04-02T05:13:10.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:09 smithi086 ceph-mon[24811]: Upgrade: Updating mon.b 2024-04-02T05:13:10.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:09 smithi086 ceph-mon[24811]: Deploying daemon mon.b on smithi192 2024-04-02T05:13:10.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:09 smithi086 ceph-mon[29505]: Upgrade: Updating mon.b 2024-04-02T05:13:10.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:09 smithi086 ceph-mon[29505]: Deploying daemon mon.b on smithi192 2024-04-02T05:13:10.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:10 smithi192 systemd[1]: Stopping Ceph mon.b for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:13:11.097 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-b[25693]: 2024-04-02T05:13:10.709+0000 7fa6929d5700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T05:13:11.097 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-b[25693]: 2024-04-02T05:13:10.709+0000 7fa6929d5700 -1 mon.b@2(peon) e3 *** Got Signal Terminated *** 2024-04-02T05:13:11.097 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:10 smithi192 podman[67833]: 2024-04-02 05:13:10.801646246 +0000 UTC m=+0.188773713 container died 4dadd8d1c3189232db61f95b603d6e03523a4af7b48ddb43b7bd995ccc27fc9b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-b, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., release=754, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, ceph=True, name=centos-stream, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD) 2024-04-02T05:13:11.405 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:11 smithi192 podman[67833]: 2024-04-02 05:13:11.193497861 +0000 UTC m=+0.580625333 container cleanup 4dadd8d1c3189232db61f95b603d6e03523a4af7b48ddb43b7bd995ccc27fc9b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-b, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, GIT_BRANCH=HEAD, release=754, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, name=centos-stream, GIT_CLEAN=True, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, vcs-type=git) 2024-04-02T05:13:11.405 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:11 smithi192 bash[67833]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-b 2024-04-02T05:13:11.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:11 smithi192 podman[67847]: 2024-04-02 05:13:11.402957316 +0000 UTC m=+0.604381281 container remove 4dadd8d1c3189232db61f95b603d6e03523a4af7b48ddb43b7bd995ccc27fc9b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-b, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, release=754, distribution-scope=public, version=8, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , io.openshift.expose-services=, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, GIT_CLEAN=True, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64) 2024-04-02T05:13:11.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:11 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mon.b.service: Deactivated successfully. 2024-04-02T05:13:11.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:11 smithi192 systemd[1]: Stopped Ceph mon.b for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:13:11.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:11 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mon.b.service: Consumed 16.969s CPU time. 2024-04-02T05:13:11.956 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:11 smithi192 systemd[1]: Starting Ceph mon.b for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:13:11.956 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:11 smithi192 podman[67981]: 2024-04-02 05:13:11.820503615 +0000 UTC m=+0.018169624 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:13:12.242 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:11 smithi192 podman[67981]: 2024-04-02 05:13:11.954022537 +0000 UTC m=+0.151688594 container create 4125e997188f34942ad2bc582ba763caca0318f503a79e2c8df6530d0b6f3c84 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-b, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, RELEASE=squid-6cec793, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-04-02T05:13:12.242 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 podman[67981]: 2024-04-02 05:13:12.156522171 +0000 UTC m=+0.354188161 container init 4125e997188f34942ad2bc582ba763caca0318f503a79e2c8df6530d0b6f3c84 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-b, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=squid-6cec793, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True) 2024-04-02T05:13:12.242 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 podman[67981]: 2024-04-02 05:13:12.159541799 +0000 UTC m=+0.357207785 container start 4125e997188f34942ad2bc582ba763caca0318f503a79e2c8df6530d0b6f3c84 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-b, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=squid-6cec793, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-04-02T05:13:12.242 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: set uid:gid to 167:167 (ceph:ceph) 2024-04-02T05:13:12.242 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev), process ceph-mon, pid 2 2024-04-02T05:13:12.242 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: pidfile_write: ignore empty --pid-file 2024-04-02T05:13:12.242 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: load: jerasure load: lrc 2024-04-02T05:13:12.242 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: RocksDB version: 7.9.2 2024-04-02T05:13:12.242 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Git sha 0 2024-04-02T05:13:12.243 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Compile date 2024-04-01 15:23:26 2024-04-02T05:13:12.243 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: DB SUMMARY 2024-04-02T05:13:12.243 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: DB Session ID: D76FRQDPXB0BPX8UEOSV 2024-04-02T05:13:12.243 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: CURRENT file: CURRENT 2024-04-02T05:13:12.243 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: IDENTITY file: IDENTITY 2024-04-02T05:13:12.243 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: MANIFEST file: MANIFEST-000009 size: 2814 Bytes 2024-04-02T05:13:12.243 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000054.sst 2024-04-02T05:13:12.243 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000052.log size: 2500545 ; 2024-04-02T05:13:12.243 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.error_if_exists: 0 2024-04-02T05:13:12.243 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.create_if_missing: 0 2024-04-02T05:13:12.243 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.paranoid_checks: 1 2024-04-02T05:13:12.243 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-02T05:13:12.244 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-02T05:13:12.244 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-02T05:13:12.244 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.env: 0x56277fb70c60 2024-04-02T05:13:12.244 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.fs: PosixFileSystem 2024-04-02T05:13:12.245 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.info_log: 0x5627812ffae0 2024-04-02T05:13:12.245 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.max_file_opening_threads: 16 2024-04-02T05:13:12.245 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.statistics: (nil) 2024-04-02T05:13:12.245 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.use_fsync: 0 2024-04-02T05:13:12.245 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.max_log_file_size: 0 2024-04-02T05:13:12.245 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-02T05:13:12.245 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-02T05:13:12.245 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.keep_log_file_num: 1000 2024-04-02T05:13:12.245 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.recycle_log_file_num: 0 2024-04-02T05:13:12.245 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.allow_fallocate: 1 2024-04-02T05:13:12.245 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.allow_mmap_reads: 0 2024-04-02T05:13:12.245 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.allow_mmap_writes: 0 2024-04-02T05:13:12.245 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.use_direct_reads: 0 2024-04-02T05:13:12.245 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-02T05:13:12.245 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.create_missing_column_families: 0 2024-04-02T05:13:12.246 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.db_log_dir: 2024-04-02T05:13:12.246 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.wal_dir: 2024-04-02T05:13:12.246 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-02T05:13:12.246 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-02T05:13:12.246 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-02T05:13:12.246 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-02T05:13:12.246 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-02T05:13:12.246 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-02T05:13:12.246 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.advise_random_on_open: 1 2024-04-02T05:13:12.247 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.db_write_buffer_size: 0 2024-04-02T05:13:12.247 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.write_buffer_manager: 0x5627812d2820 2024-04-02T05:13:12.247 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-02T05:13:12.247 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-02T05:13:12.247 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-02T05:13:12.247 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.rate_limiter: (nil) 2024-04-02T05:13:12.247 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-02T05:13:12.247 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.wal_recovery_mode: 2 2024-04-02T05:13:12.247 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.enable_thread_tracking: 0 2024-04-02T05:13:12.247 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.enable_pipelined_write: 0 2024-04-02T05:13:12.247 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.unordered_write: 0 2024-04-02T05:13:12.248 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-02T05:13:12.248 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-02T05:13:12.248 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-02T05:13:12.248 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-02T05:13:12.248 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.row_cache: None 2024-04-02T05:13:12.248 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.wal_filter: None 2024-04-02T05:13:12.248 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-02T05:13:12.248 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.allow_ingest_behind: 0 2024-04-02T05:13:12.248 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.two_write_queues: 0 2024-04-02T05:13:12.248 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.manual_wal_flush: 0 2024-04-02T05:13:12.249 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.wal_compression: 0 2024-04-02T05:13:12.249 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.atomic_flush: 0 2024-04-02T05:13:12.249 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-02T05:13:12.249 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-02T05:13:12.249 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-02T05:13:12.249 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.log_readahead_size: 0 2024-04-02T05:13:12.249 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-02T05:13:12.249 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.best_efforts_recovery: 0 2024-04-02T05:13:12.249 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-02T05:13:12.249 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-02T05:13:12.249 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.allow_data_in_errors: 0 2024-04-02T05:13:12.249 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.db_host_id: __hostname__ 2024-04-02T05:13:12.249 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-02T05:13:12.249 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.max_background_jobs: 2 2024-04-02T05:13:12.250 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.max_background_compactions: -1 2024-04-02T05:13:12.250 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.max_subcompactions: 1 2024-04-02T05:13:12.250 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-02T05:13:12.250 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-02T05:13:12.250 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-02T05:13:12.250 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.max_total_wal_size: 0 2024-04-02T05:13:12.250 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-02T05:13:12.250 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-02T05:13:12.250 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-02T05:13:12.250 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-02T05:13:12.250 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.max_open_files: -1 2024-04-02T05:13:12.250 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.bytes_per_sync: 0 2024-04-02T05:13:12.250 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-02T05:13:12.250 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-02T05:13:12.250 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.compaction_readahead_size: 0 2024-04-02T05:13:12.251 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.max_background_flushes: -1 2024-04-02T05:13:12.251 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Compression algorithms supported: 2024-04-02T05:13:12.251 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-02T05:13:12.251 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: kZSTD supported: 0 2024-04-02T05:13:12.251 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: kXpressCompression supported: 0 2024-04-02T05:13:12.251 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: kLZ4HCCompression supported: 1 2024-04-02T05:13:12.251 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: kZlibCompression supported: 1 2024-04-02T05:13:12.251 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: kSnappyCompression supported: 1 2024-04-02T05:13:12.251 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: kLZ4Compression supported: 1 2024-04-02T05:13:12.251 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: kBZip2Compression supported: 0 2024-04-02T05:13:12.251 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-02T05:13:12.251 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-02T05:13:12.251 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-04-02T05:13:12.251 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-02T05:13:12.252 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-02T05:13:12.252 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.merge_operator: 2024-04-02T05:13:12.252 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.compaction_filter: None 2024-04-02T05:13:12.252 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.compaction_filter_factory: None 2024-04-02T05:13:12.252 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.sst_partitioner_factory: None 2024-04-02T05:13:12.252 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-02T05:13:12.252 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-02T05:13:12.252 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5627812ffbe0) 2024-04-02T05:13:12.252 INFO:journalctl@ceph.mon.b.smithi192.stdout: cache_index_and_filter_blocks: 1 2024-04-02T05:13:12.252 INFO:journalctl@ceph.mon.b.smithi192.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-02T05:13:12.252 INFO:journalctl@ceph.mon.b.smithi192.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-02T05:13:12.252 INFO:journalctl@ceph.mon.b.smithi192.stdout: pin_top_level_index_and_filter: 1 2024-04-02T05:13:12.252 INFO:journalctl@ceph.mon.b.smithi192.stdout: index_type: 0 2024-04-02T05:13:12.252 INFO:journalctl@ceph.mon.b.smithi192.stdout: data_block_index_type: 0 2024-04-02T05:13:12.253 INFO:journalctl@ceph.mon.b.smithi192.stdout: index_shortening: 1 2024-04-02T05:13:12.253 INFO:journalctl@ceph.mon.b.smithi192.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-02T05:13:12.253 INFO:journalctl@ceph.mon.b.smithi192.stdout: checksum: 4 2024-04-02T05:13:12.253 INFO:journalctl@ceph.mon.b.smithi192.stdout: no_block_cache: 0 2024-04-02T05:13:12.253 INFO:journalctl@ceph.mon.b.smithi192.stdout: block_cache: 0x562781303090 2024-04-02T05:13:12.253 INFO:journalctl@ceph.mon.b.smithi192.stdout: block_cache_name: BinnedLRUCache 2024-04-02T05:13:12.253 INFO:journalctl@ceph.mon.b.smithi192.stdout: block_cache_options: 2024-04-02T05:13:12.253 INFO:journalctl@ceph.mon.b.smithi192.stdout: capacity : 536870912 2024-04-02T05:13:12.253 INFO:journalctl@ceph.mon.b.smithi192.stdout: num_shard_bits : 4 2024-04-02T05:13:12.253 INFO:journalctl@ceph.mon.b.smithi192.stdout: strict_capacity_limit : 0 2024-04-02T05:13:12.253 INFO:journalctl@ceph.mon.b.smithi192.stdout: high_pri_pool_ratio: 0.000 2024-04-02T05:13:12.254 INFO:journalctl@ceph.mon.b.smithi192.stdout: block_cache_compressed: (nil) 2024-04-02T05:13:12.254 INFO:journalctl@ceph.mon.b.smithi192.stdout: persistent_cache: (nil) 2024-04-02T05:13:12.254 INFO:journalctl@ceph.mon.b.smithi192.stdout: block_size: 4096 2024-04-02T05:13:12.254 INFO:journalctl@ceph.mon.b.smithi192.stdout: block_size_deviation: 10 2024-04-02T05:13:12.254 INFO:journalctl@ceph.mon.b.smithi192.stdout: block_restart_interval: 16 2024-04-02T05:13:12.254 INFO:journalctl@ceph.mon.b.smithi192.stdout: index_block_restart_interval: 1 2024-04-02T05:13:12.254 INFO:journalctl@ceph.mon.b.smithi192.stdout: metadata_block_size: 4096 2024-04-02T05:13:12.254 INFO:journalctl@ceph.mon.b.smithi192.stdout: partition_filters: 0 2024-04-02T05:13:12.254 INFO:journalctl@ceph.mon.b.smithi192.stdout: use_delta_encoding: 1 2024-04-02T05:13:12.254 INFO:journalctl@ceph.mon.b.smithi192.stdout: filter_policy: bloomfilter 2024-04-02T05:13:12.254 INFO:journalctl@ceph.mon.b.smithi192.stdout: whole_key_filtering: 1 2024-04-02T05:13:12.254 INFO:journalctl@ceph.mon.b.smithi192.stdout: verify_compression: 0 2024-04-02T05:13:12.255 INFO:journalctl@ceph.mon.b.smithi192.stdout: read_amp_bytes_per_bit: 0 2024-04-02T05:13:12.255 INFO:journalctl@ceph.mon.b.smithi192.stdout: format_version: 5 2024-04-02T05:13:12.255 INFO:journalctl@ceph.mon.b.smithi192.stdout: enable_index_compression: 1 2024-04-02T05:13:12.255 INFO:journalctl@ceph.mon.b.smithi192.stdout: block_align: 0 2024-04-02T05:13:12.255 INFO:journalctl@ceph.mon.b.smithi192.stdout: max_auto_readahead_size: 262144 2024-04-02T05:13:12.255 INFO:journalctl@ceph.mon.b.smithi192.stdout: prepopulate_block_cache: 0 2024-04-02T05:13:12.255 INFO:journalctl@ceph.mon.b.smithi192.stdout: initial_auto_readahead_size: 8192 2024-04-02T05:13:12.255 INFO:journalctl@ceph.mon.b.smithi192.stdout: num_file_reads_for_auto_readahead: 2 2024-04-02T05:13:12.255 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.write_buffer_size: 33554432 2024-04-02T05:13:12.255 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.max_write_buffer_number: 2 2024-04-02T05:13:12.255 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.compression: NoCompression 2024-04-02T05:13:12.255 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.bottommost_compression: Disabled 2024-04-02T05:13:12.255 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.prefix_extractor: nullptr 2024-04-02T05:13:12.255 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-02T05:13:12.255 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.num_levels: 7 2024-04-02T05:13:12.256 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-02T05:13:12.256 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-02T05:13:12.256 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-02T05:13:12.256 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-02T05:13:12.256 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-02T05:13:12.256 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-02T05:13:12.256 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-02T05:13:12.256 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-02T05:13:12.256 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-02T05:13:12.256 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-02T05:13:12.256 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-02T05:13:12.256 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-02T05:13:12.256 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-02T05:13:12.256 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.compression_opts.level: 32767 2024-04-02T05:13:12.256 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.compression_opts.strategy: 0 2024-04-02T05:13:12.257 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-02T05:13:12.257 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-02T05:13:12.257 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-02T05:13:12.257 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-02T05:13:12.257 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.compression_opts.enabled: false 2024-04-02T05:13:12.257 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-02T05:13:12.257 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-02T05:13:12.257 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-02T05:13:12.257 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-02T05:13:12.257 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.target_file_size_base: 67108864 2024-04-02T05:13:12.258 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-02T05:13:12.258 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-02T05:13:12.258 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-02T05:13:12.258 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-02T05:13:12.258 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-02T05:13:12.258 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-02T05:13:12.258 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-02T05:13:12.258 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-02T05:13:12.258 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-02T05:13:12.258 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-02T05:13:12.258 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-02T05:13:12.259 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-02T05:13:12.259 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-02T05:13:12.259 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-02T05:13:12.259 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.arena_block_size: 1048576 2024-04-02T05:13:12.259 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-02T05:13:12.259 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-02T05:13:12.259 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.disable_auto_compactions: 0 2024-04-02T05:13:12.259 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-02T05:13:12.259 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-02T05:13:12.259 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-02T05:13:12.259 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-02T05:13:12.259 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-02T05:13:12.259 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-02T05:13:12.259 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-02T05:13:12.260 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-02T05:13:12.260 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-02T05:13:12.260 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-02T05:13:12.260 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.table_properties_collectors: 2024-04-02T05:13:12.260 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.inplace_update_support: 0 2024-04-02T05:13:12.260 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-02T05:13:12.260 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-02T05:13:12.260 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-02T05:13:12.260 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-02T05:13:12.260 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.bloom_locality: 0 2024-04-02T05:13:12.260 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.max_successive_merges: 0 2024-04-02T05:13:12.260 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-02T05:13:12.260 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.paranoid_file_checks: 0 2024-04-02T05:13:12.261 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.force_consistency_checks: 1 2024-04-02T05:13:12.261 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.report_bg_io_stats: 0 2024-04-02T05:13:12.261 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.ttl: 2592000 2024-04-02T05:13:12.262 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-02T05:13:12.262 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-02T05:13:12.262 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-02T05:13:12.262 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.enable_blob_files: false 2024-04-02T05:13:12.262 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.min_blob_size: 0 2024-04-02T05:13:12.262 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.blob_file_size: 268435456 2024-04-02T05:13:12.262 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-02T05:13:12.262 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-02T05:13:12.262 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-02T05:13:12.262 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-02T05:13:12.262 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-02T05:13:12.262 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.blob_file_starting_level: 0 2024-04-02T05:13:12.262 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-02T05:13:12.262 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 54.sst 2024-04-02T05:13:12.262 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-02T05:13:12.263 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 56, last_sequence is 28744, log_number is 52,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-02T05:13:12.263 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 52 2024-04-02T05:13:12.263 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 014d3486-47e5-4d5c-a03b-1bba1c798948 2024-04-02T05:13:12.263 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712034792188810, "job": 1, "event": "recovery_started", "wal_files": [52]} 2024-04-02T05:13:12.263 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #52 mode 2 2024-04-02T05:13:12.263 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712034792195418, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 57, "file_size": 1528943, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 28749, "largest_seqno": 30184, "table_properties": {"data_size": 1523111, "index_size": 3345, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1477, "raw_key_size": 14031, "raw_average_key_size": 24, "raw_value_size": 1511049, "raw_average_value_size": 2623, "num_data_blocks": 149, "num_entries": 576, "num_filter_entries": 576, "num_deletions": 7, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1712034792, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "014d3486-47e5-4d5c-a03b-1bba1c798948", "db_session_id": "D76FRQDPXB0BPX8UEOSV", "orig_file_number": 57, "seqno_to_time_mapping": "N/A"}} 2024-04-02T05:13:12.263 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712034792195510, "job": 1, "event": "recovery_finished"} 2024-04-02T05:13:12.263 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: [db/version_set.cc:5047] Creating manifest 59 2024-04-02T05:13:12.263 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-02T05:13:12.263 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000052.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-02T05:13:12.263 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5627813f4000 2024-04-02T05:13:12.263 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: DB pointer 0x5627813de000 2024-04-02T05:13:12.263 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-02T05:13:12.263 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-02T05:13:12.264 INFO:journalctl@ceph.mon.b.smithi192.stdout: ** DB Stats ** 2024-04-02T05:13:12.264 INFO:journalctl@ceph.mon.b.smithi192.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-02T05:13:12.264 INFO:journalctl@ceph.mon.b.smithi192.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-02T05:13:12.264 INFO:journalctl@ceph.mon.b.smithi192.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-02T05:13:12.264 INFO:journalctl@ceph.mon.b.smithi192.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-02T05:13:12.264 INFO:journalctl@ceph.mon.b.smithi192.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-02T05:13:12.264 INFO:journalctl@ceph.mon.b.smithi192.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-02T05:13:12.264 INFO:journalctl@ceph.mon.b.smithi192.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-02T05:13:12.264 INFO:journalctl@ceph.mon.b.smithi192.stdout: 2024-04-02T05:13:12.264 INFO:journalctl@ceph.mon.b.smithi192.stdout: ** Compaction Stats [default] ** 2024-04-02T05:13:12.264 INFO:journalctl@ceph.mon.b.smithi192.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-02T05:13:12.265 INFO:journalctl@ceph.mon.b.smithi192.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-02T05:13:12.265 INFO:journalctl@ceph.mon.b.smithi192.stdout: L0 1/0 1.46 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 494.9 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-02T05:13:12.265 INFO:journalctl@ceph.mon.b.smithi192.stdout: L6 1/0 9.10 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-02T05:13:12.265 INFO:journalctl@ceph.mon.b.smithi192.stdout: Sum 2/0 10.56 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 494.9 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-02T05:13:12.265 INFO:journalctl@ceph.mon.b.smithi192.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 494.9 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-02T05:13:12.265 INFO:journalctl@ceph.mon.b.smithi192.stdout: 2024-04-02T05:13:12.265 INFO:journalctl@ceph.mon.b.smithi192.stdout: ** Compaction Stats [default] ** 2024-04-02T05:13:12.265 INFO:journalctl@ceph.mon.b.smithi192.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-02T05:13:12.265 INFO:journalctl@ceph.mon.b.smithi192.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T05:13:12.265 INFO:journalctl@ceph.mon.b.smithi192.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 494.9 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-02T05:13:12.265 INFO:journalctl@ceph.mon.b.smithi192.stdout: 2024-04-02T05:13:12.265 INFO:journalctl@ceph.mon.b.smithi192.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-02T05:13:12.265 INFO:journalctl@ceph.mon.b.smithi192.stdout: 2024-04-02T05:13:12.265 INFO:journalctl@ceph.mon.b.smithi192.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-02T05:13:12.266 INFO:journalctl@ceph.mon.b.smithi192.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-04-02T05:13:12.266 INFO:journalctl@ceph.mon.b.smithi192.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-02T05:13:12.266 INFO:journalctl@ceph.mon.b.smithi192.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-02T05:13:12.266 INFO:journalctl@ceph.mon.b.smithi192.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-02T05:13:12.266 INFO:journalctl@ceph.mon.b.smithi192.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-02T05:13:12.266 INFO:journalctl@ceph.mon.b.smithi192.stdout: Cumulative compaction: 0.00 GB write, 159.37 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T05:13:12.266 INFO:journalctl@ceph.mon.b.smithi192.stdout: Interval compaction: 0.00 GB write, 159.37 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T05:13:12.266 INFO:journalctl@ceph.mon.b.smithi192.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-02T05:13:12.266 INFO:journalctl@ceph.mon.b.smithi192.stdout: Block cache BinnedLRUCache@0x562781303090#2 capacity: 512.00 MB usage: 5.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 9e-06 secs_since: 0 2024-04-02T05:13:12.267 INFO:journalctl@ceph.mon.b.smithi192.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.53 KB,0.000292063%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2024-04-02T05:13:12.267 INFO:journalctl@ceph.mon.b.smithi192.stdout: 2024-04-02T05:13:12.267 INFO:journalctl@ceph.mon.b.smithi192.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-02T05:13:12.267 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: starting mon.b rank 2 at public addrs [v2:172.21.15.192:3300/0,v1:172.21.15.192:6789/0] at bind addrs [v2:172.21.15.192:3300/0,v1:172.21.15.192:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 2024-04-02T05:13:12.267 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: mon.b@-1(???) e3 preinit fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 2024-04-02T05:13:12.267 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: mon.b@-1(???).mds e1 new map 2024-04-02T05:13:12.267 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: mon.b@-1(???).mds e1 print_map 2024-04-02T05:13:12.267 INFO:journalctl@ceph.mon.b.smithi192.stdout: e1 2024-04-02T05:13:12.267 INFO:journalctl@ceph.mon.b.smithi192.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-02T05:13:12.267 INFO:journalctl@ceph.mon.b.smithi192.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-02T05:13:12.267 INFO:journalctl@ceph.mon.b.smithi192.stdout: legacy client fscid: -1 2024-04-02T05:13:12.267 INFO:journalctl@ceph.mon.b.smithi192.stdout: 2024-04-02T05:13:12.267 INFO:journalctl@ceph.mon.b.smithi192.stdout: No filesystems configured 2024-04-02T05:13:12.267 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: mon.b@-1(???).osd e110 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-02T05:13:12.267 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: mon.b@-1(???).osd e110 crush map has features 432629239337189376, adjusting msgr requires 2024-04-02T05:13:12.268 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: mon.b@-1(???).osd e110 crush map has features 432629239337189376, adjusting msgr requires 2024-04-02T05:13:12.268 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: mon.b@-1(???).osd e110 crush map has features 432629239337189376, adjusting msgr requires 2024-04-02T05:13:12.268 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 ceph-mon[68000]: mon.b@-1(???).paxosservice(auth 1..26) refresh upgraded, format 0 -> 3 2024-04-02T05:13:12.268 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 bash[67981]: 4125e997188f34942ad2bc582ba763caca0318f503a79e2c8df6530d0b6f3c84 2024-04-02T05:13:12.268 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:12 smithi192 systemd[1]: Started Ceph mon.b for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:13:13.478 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:13 smithi192 ceph-mon[68000]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:13.478 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:13 smithi192 ceph-mon[68000]: mon.b calling monitor election 2024-04-02T05:13:13.479 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:13 smithi192 ceph-mon[68000]: mon.a calling monitor election 2024-04-02T05:13:13.479 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:13 smithi192 ceph-mon[68000]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-02T05:13:13.479 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:13 smithi192 ceph-mon[68000]: monmap e3: 3 mons at {a=[v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0],b=[v2:172.21.15.192:3300/0,v1:172.21.15.192:6789/0],c=[v2:172.21.15.86:3301/0,v1:172.21.15.86:6790/0]} 2024-04-02T05:13:13.479 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:13 smithi192 ceph-mon[68000]: fsmap 2024-04-02T05:13:13.479 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:13 smithi192 ceph-mon[68000]: osdmap e110: 8 total, 8 up, 8 in 2024-04-02T05:13:13.479 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:13 smithi192 ceph-mon[68000]: mgrmap e41: y(active, since 2m), standbys: x 2024-04-02T05:13:13.479 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:13 smithi192 ceph-mon[68000]: overall HEALTH_OK 2024-04-02T05:13:13.479 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:13 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:13.479 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:13 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:13.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:13:13 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:13:13] "GET /metrics HTTP/1.1" 200 37991 "" "Prometheus/2.43.0" 2024-04-02T05:13:13.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:13 smithi086 ceph-mon[29505]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:13.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:13 smithi086 ceph-mon[29505]: mon.b calling monitor election 2024-04-02T05:13:13.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:13 smithi086 ceph-mon[29505]: mon.a calling monitor election 2024-04-02T05:13:13.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:13 smithi086 ceph-mon[29505]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-02T05:13:13.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:13 smithi086 ceph-mon[29505]: monmap e3: 3 mons at {a=[v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0],b=[v2:172.21.15.192:3300/0,v1:172.21.15.192:6789/0],c=[v2:172.21.15.86:3301/0,v1:172.21.15.86:6790/0]} 2024-04-02T05:13:13.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:13 smithi086 ceph-mon[29505]: fsmap 2024-04-02T05:13:13.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:13 smithi086 ceph-mon[29505]: osdmap e110: 8 total, 8 up, 8 in 2024-04-02T05:13:13.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:13 smithi086 ceph-mon[29505]: mgrmap e41: y(active, since 2m), standbys: x 2024-04-02T05:13:13.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:13 smithi086 ceph-mon[29505]: overall HEALTH_OK 2024-04-02T05:13:13.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:13 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:13.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:13 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:13.837 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:13 smithi086 ceph-mon[24811]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:13.837 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:13 smithi086 ceph-mon[24811]: mon.b calling monitor election 2024-04-02T05:13:13.837 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:13 smithi086 ceph-mon[24811]: mon.a calling monitor election 2024-04-02T05:13:13.837 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:13 smithi086 ceph-mon[24811]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-02T05:13:13.837 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:13 smithi086 ceph-mon[24811]: monmap e3: 3 mons at {a=[v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0],b=[v2:172.21.15.192:3300/0,v1:172.21.15.192:6789/0],c=[v2:172.21.15.86:3301/0,v1:172.21.15.86:6790/0]} 2024-04-02T05:13:13.838 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:13 smithi086 ceph-mon[24811]: fsmap 2024-04-02T05:13:13.838 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:13 smithi086 ceph-mon[24811]: osdmap e110: 8 total, 8 up, 8 in 2024-04-02T05:13:13.838 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:13 smithi086 ceph-mon[24811]: mgrmap e41: y(active, since 2m), standbys: x 2024-04-02T05:13:13.838 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:13 smithi086 ceph-mon[24811]: overall HEALTH_OK 2024-04-02T05:13:13.838 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:13 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:13.839 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:13 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:14.671 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:14 smithi192 ceph-mon[68000]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:13:14.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:14 smithi086 ceph-mon[24811]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:13:14.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:14 smithi086 ceph-mon[29505]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:13:17.333 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:13:17 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:13:17.203+0000 7f180b134700 -1 mgr.server handle_report got status from non-daemon mon.b 2024-04-02T05:13:17.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:16 smithi086 ceph-mon[24811]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-02T05:13:17.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:16 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:17.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:16 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:17.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:16 smithi086 ceph-mon[29505]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-02T05:13:17.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:16 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:17.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:16 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:17.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:16 smithi192 ceph-mon[68000]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-02T05:13:17.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:16 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:17.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:16 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:18.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:18 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:18.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:18 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:18.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:18 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:13:18.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:18 smithi192 ceph-mon[68000]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-02T05:13:19.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:18 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:19.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:18 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:19.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:18 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:13:19.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:18 smithi086 ceph-mon[24811]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-02T05:13:19.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:18 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:19.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:18 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:19.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:18 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:13:19.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:18 smithi086 ceph-mon[29505]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-02T05:13:21.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:21 smithi086 ceph-mon[24811]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:13:21.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:21 smithi086 ceph-mon[24811]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:13:21.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:21 smithi086 ceph-mon[29505]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:13:21.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:21 smithi086 ceph-mon[29505]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:13:21.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:21 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:13:21.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:21 smithi192 ceph-mon[68000]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:13:22.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: Detected new or changed devices on smithi192 2024-04-02T05:13:22.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:22.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:22.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:13:22.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:13:22.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:22.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:13:22.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:22.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:22.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:22.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:22.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-02T05:13:22.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:22.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:22.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:22.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.336 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: Detected new or changed devices on smithi192 2024-04-02T05:13:22.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:22.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:22.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:13:22.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:13:22.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:22.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:13:22.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.337 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:22.337 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.337 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:22.337 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.337 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:22.337 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.337 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:22.337 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.337 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.337 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-02T05:13:22.337 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.337 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:22.337 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.337 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:22.337 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.338 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:22.338 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.338 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.338 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.338 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.338 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.338 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.338 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.338 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.338 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.338 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.339 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.339 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.339 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.339 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.339 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.339 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.339 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.339 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.339 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:13:22.339 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:13:22.339 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-02T05:13:22.339 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.339 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.339 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-02T05:13:22.340 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:13:22.340 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:13:22.340 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-02T05:13:22.340 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:13:22.340 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:13:22.340 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:13:22.340 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:13:22.340 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.340 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.340 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.340 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.340 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.341 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.341 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.341 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.341 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.341 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.341 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.341 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:13:22.341 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:13:22.341 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-02T05:13:22.341 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.342 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.342 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-02T05:13:22.342 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:13:22.342 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:13:22.342 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-02T05:13:22.342 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:13:22.342 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:13:22.342 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:13:22.342 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:13:22.342 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-02T05:13:22.343 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:13:22.343 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:13:22.343 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:13:22.343 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:13:22.343 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-02T05:13:22.343 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.343 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.343 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:13:22.343 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:13:22.343 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-02T05:13:22.343 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:13:22.343 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:13:22.343 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:13:22.343 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:13:22.343 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-02T05:13:22.344 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:13:22.344 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:13:22.344 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-02T05:13:22.344 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.344 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.344 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.344 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.344 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.344 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-02T05:13:22.344 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:13:22.345 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:13:22.345 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:13:22.345 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:13:22.345 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-02T05:13:22.345 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.345 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.345 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:13:22.345 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:13:22.345 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-02T05:13:22.345 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:13:22.345 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:13:22.345 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:13:22.345 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:13:22.346 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-02T05:13:22.346 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:13:22.346 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:13:22.346 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-02T05:13:22.346 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.346 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.346 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.346 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.346 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.346 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.346 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.347 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.347 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.347 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.347 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.347 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.347 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:13:22.347 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:13:22.347 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-02T05:13:22.347 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:13:22.347 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:13:22.347 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:22.347 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:13:22.347 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:13:22.347 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:22.348 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.348 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.348 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.348 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.348 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.348 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.348 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.348 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:13:22.348 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:13:22.348 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-02T05:13:22.348 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:13:22.348 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:13:22.348 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:22.349 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:13:22.349 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:13:22.349 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:22 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:22.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: Detected new or changed devices on smithi192 2024-04-02T05:13:22.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:22.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:22.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:13:22.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:13:22.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:22.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:13:22.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:22.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:22.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:22.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:22.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-02T05:13:22.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:22.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:22.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:22.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:22.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-02T05:13:22.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:13:22.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:13:22.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-02T05:13:22.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-02T05:13:22.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:13:22.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:13:22.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-02T05:13:22.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:13:22.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:13:22.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:13:22.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:13:22.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-02T05:13:22.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:13:22.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:13:22.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:13:22.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:13:22.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-02T05:13:22.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:13:22.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:13:22.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-02T05:13:22.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:13:22.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:13:22.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:13:22.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:13:22.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-02T05:13:22.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:13:22.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:13:22.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-02T05:13:22.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.434 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.434 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.434 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.434 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.435 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.435 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.435 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:13:22.435 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:13:22.435 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:13:22.435 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-02T05:13:22.435 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:13:22.435 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:13:22.435 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:22.436 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:13:22.436 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:13:22.436 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:22 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:23.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:23 smithi086 ceph-mon[24811]: Upgrade: Setting container_image for all crash 2024-04-02T05:13:23.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:23 smithi086 ceph-mon[24811]: Upgrade: Setting container_image for all mds 2024-04-02T05:13:23.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:23 smithi086 ceph-mon[24811]: Upgrade: Setting container_image for all rbd-mirror 2024-04-02T05:13:23.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:23 smithi086 ceph-mon[24811]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-02T05:13:23.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:23 smithi086 ceph-mon[24811]: Upgrade: Setting container_image for all ceph-exporter 2024-04-02T05:13:23.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:23 smithi086 ceph-mon[24811]: Upgrade: Setting container_image for all nfs 2024-04-02T05:13:23.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:23 smithi086 ceph-mon[24811]: Upgrade: Setting container_image for all nvmeof 2024-04-02T05:13:23.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:23 smithi086 ceph-mon[24811]: Upgrade: Setting container_image for all node-exporter 2024-04-02T05:13:23.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:23 smithi086 ceph-mon[24811]: Upgrade: Setting container_image for all prometheus 2024-04-02T05:13:23.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:23 smithi086 ceph-mon[24811]: Upgrade: Setting container_image for all alertmanager 2024-04-02T05:13:23.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:23 smithi086 ceph-mon[24811]: Upgrade: Setting container_image for all grafana 2024-04-02T05:13:23.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:23 smithi086 ceph-mon[24811]: Upgrade: Setting container_image for all loki 2024-04-02T05:13:23.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:23 smithi086 ceph-mon[24811]: Upgrade: Setting container_image for all promtail 2024-04-02T05:13:23.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:23 smithi086 ceph-mon[24811]: Upgrade: Finalizing container_image settings 2024-04-02T05:13:23.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:23 smithi086 ceph-mon[24811]: Upgrade: Complete! 2024-04-02T05:13:23.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:23 smithi086 ceph-mon[24811]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-02T05:13:23.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:23 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:23.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:23 smithi086 ceph-mon[29505]: Upgrade: Setting container_image for all crash 2024-04-02T05:13:23.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:23 smithi086 ceph-mon[29505]: Upgrade: Setting container_image for all mds 2024-04-02T05:13:23.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:23 smithi086 ceph-mon[29505]: Upgrade: Setting container_image for all rbd-mirror 2024-04-02T05:13:23.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:23 smithi086 ceph-mon[29505]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-02T05:13:23.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:23 smithi086 ceph-mon[29505]: Upgrade: Setting container_image for all ceph-exporter 2024-04-02T05:13:23.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:23 smithi086 ceph-mon[29505]: Upgrade: Setting container_image for all nfs 2024-04-02T05:13:23.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:23 smithi086 ceph-mon[29505]: Upgrade: Setting container_image for all nvmeof 2024-04-02T05:13:23.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:23 smithi086 ceph-mon[29505]: Upgrade: Setting container_image for all node-exporter 2024-04-02T05:13:23.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:23 smithi086 ceph-mon[29505]: Upgrade: Setting container_image for all prometheus 2024-04-02T05:13:23.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:23 smithi086 ceph-mon[29505]: Upgrade: Setting container_image for all alertmanager 2024-04-02T05:13:23.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:23 smithi086 ceph-mon[29505]: Upgrade: Setting container_image for all grafana 2024-04-02T05:13:23.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:23 smithi086 ceph-mon[29505]: Upgrade: Setting container_image for all loki 2024-04-02T05:13:23.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:23 smithi086 ceph-mon[29505]: Upgrade: Setting container_image for all promtail 2024-04-02T05:13:23.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:23 smithi086 ceph-mon[29505]: Upgrade: Finalizing container_image settings 2024-04-02T05:13:23.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:23 smithi086 ceph-mon[29505]: Upgrade: Complete! 2024-04-02T05:13:23.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:23 smithi086 ceph-mon[29505]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-02T05:13:23.336 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:23 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:23.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:23 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all crash 2024-04-02T05:13:23.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:23 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all mds 2024-04-02T05:13:23.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:23 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all rbd-mirror 2024-04-02T05:13:23.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:23 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-02T05:13:23.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:23 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all ceph-exporter 2024-04-02T05:13:23.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:23 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all nfs 2024-04-02T05:13:23.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:23 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all nvmeof 2024-04-02T05:13:23.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:23 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all node-exporter 2024-04-02T05:13:23.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:23 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all prometheus 2024-04-02T05:13:23.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:23 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all alertmanager 2024-04-02T05:13:23.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:23 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all grafana 2024-04-02T05:13:23.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:23 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all loki 2024-04-02T05:13:23.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:23 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all promtail 2024-04-02T05:13:23.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:23 smithi192 ceph-mon[68000]: Upgrade: Finalizing container_image settings 2024-04-02T05:13:23.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:23 smithi192 ceph-mon[68000]: Upgrade: Complete! 2024-04-02T05:13:23.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:23 smithi192 ceph-mon[68000]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-02T05:13:23.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:23 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:23.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:13:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:13:23] "GET /metrics HTTP/1.1" 200 38002 "" "Prometheus/2.43.0" 2024-04-02T05:13:25.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:25 smithi086 ceph-mon[24811]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:13:25.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:25 smithi086 ceph-mon[29505]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:13:25.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:25 smithi192 ceph-mon[68000]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:13:27.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:26 smithi086 ceph-mon[24811]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:27.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:26 smithi086 ceph-mon[29505]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:27.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:26 smithi192 ceph-mon[68000]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:29.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:28 smithi086 ceph-mon[24811]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:29.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:28 smithi086 ceph-mon[29505]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:29.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:28 smithi192 ceph-mon[68000]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:31.229 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:30 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:13:31.230 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:30 smithi192 ceph-mon[68000]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:13:31.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:30 smithi086 ceph-mon[24811]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:13:31.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:30 smithi086 ceph-mon[24811]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:13:31.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:30 smithi086 ceph-mon[29505]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:13:31.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:30 smithi086 ceph-mon[29505]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:13:33.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:32 smithi086 ceph-mon[24811]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:33.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:32 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:13:33.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:32 smithi086 ceph-mon[29505]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:33.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:32 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:13:33.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:32 smithi192 ceph-mon[68000]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:33.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:32 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:13:33.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:13:33 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:13:33] "GET /metrics HTTP/1.1" 200 38001 "" "Prometheus/2.43.0" 2024-04-02T05:13:35.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:34 smithi086 ceph-mon[29505]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:13:35.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:34 smithi086 ceph-mon[24811]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:13:35.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:34 smithi192 ceph-mon[68000]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:13:37.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:36 smithi086 ceph-mon[29505]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:37.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:36 smithi086 ceph-mon[24811]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:37.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:36 smithi192 ceph-mon[68000]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:38.924 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch ps' 2024-04-02T05:13:39.233 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:38 smithi086 ceph-mon[24811]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:39.233 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:38 smithi086 ceph-mon[29505]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:39.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:38 smithi192 ceph-mon[68000]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:40.198 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:39 smithi086 ceph-mon[29505]: from='client.15834 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:13:40.198 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:39 smithi086 ceph-mon[24811]: from='client.15834 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:13:40.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:39 smithi192 ceph-mon[68000]: from='client.15834 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:13:40.874 INFO:teuthology.orchestra.run.smithi086.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T05:13:40.875 INFO:teuthology.orchestra.run.smithi086.stdout:alertmanager.a smithi086 *:9093,9094 running (105s) 66s ago 23m 16.0M - 0.25.0 c8568f914cd2 660ea62bb6b9 2024-04-02T05:13:40.875 INFO:teuthology.orchestra.run.smithi086.stdout:grafana.a smithi192 *:3000 running (87s) 24s ago 23m 74.7M - 9.4.12 2bacad6d85d8 259af9e63aa5 2024-04-02T05:13:40.875 INFO:teuthology.orchestra.run.smithi086.stdout:iscsi.foo.smithi086.rsbqwx smithi086 running (71s) 66s ago 22m 52.0M - 3.5 e1d6a67b021e fe728eb940f7 2024-04-02T05:13:40.875 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.x smithi192 *:8443,9283,8765 running (2m) 24s ago 28m 450M - 19.0.0-1979-g6cec793d 78f830190839 dd1e6d0f0a95 2024-04-02T05:13:40.875 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.y smithi086 *:8443,9283,8765 running (12m) 66s ago 29m 510M - 19.0.0-1979-g6cec793d 78f830190839 3dca54a018ab 2024-04-02T05:13:40.875 INFO:teuthology.orchestra.run.smithi086.stdout:mon.a smithi086 running (29m) 66s ago 29m 79.8M 2048M 17.2.0 e1d6a67b021e b6c507ab96f8 2024-04-02T05:13:40.875 INFO:teuthology.orchestra.run.smithi086.stdout:mon.b smithi192 running (29s) 24s ago 28m 18.6M 2048M 19.0.0-1979-g6cec793d 78f830190839 4125e997188f 2024-04-02T05:13:40.875 INFO:teuthology.orchestra.run.smithi086.stdout:mon.c smithi086 running (28m) 66s ago 28m 58.3M 2048M 17.2.0 e1d6a67b021e 49275483a2d4 2024-04-02T05:13:40.875 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.a smithi086 *:9100 running (2m) 66s ago 24m 15.8M - 1.5.0 0da6a335fe13 05a844569d7b 2024-04-02T05:13:40.875 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.b smithi192 *:9100 running (2m) 24s ago 24m 17.0M - 1.5.0 0da6a335fe13 1a7d6788d58b 2024-04-02T05:13:40.875 INFO:teuthology.orchestra.run.smithi086.stdout:osd.0 smithi086 running (27m) 66s ago 27m 57.5M 2992M 17.2.0 e1d6a67b021e 6d90c60d1c71 2024-04-02T05:13:40.875 INFO:teuthology.orchestra.run.smithi086.stdout:osd.1 smithi086 running (27m) 66s ago 27m 56.5M 2992M 17.2.0 e1d6a67b021e 203689f71603 2024-04-02T05:13:40.876 INFO:teuthology.orchestra.run.smithi086.stdout:osd.2 smithi086 running (26m) 66s ago 26m 50.4M 2992M 17.2.0 e1d6a67b021e 472c860a9af8 2024-04-02T05:13:40.876 INFO:teuthology.orchestra.run.smithi086.stdout:osd.3 smithi086 running (26m) 66s ago 26m 52.7M 2992M 17.2.0 e1d6a67b021e 327538dac44e 2024-04-02T05:13:40.876 INFO:teuthology.orchestra.run.smithi086.stdout:osd.4 smithi192 running (26m) 24s ago 26m 56.0M 3248M 17.2.0 e1d6a67b021e 6535d7919a2d 2024-04-02T05:13:40.876 INFO:teuthology.orchestra.run.smithi086.stdout:osd.5 smithi192 running (25m) 24s ago 25m 54.9M 3248M 17.2.0 e1d6a67b021e 7259249605f2 2024-04-02T05:13:40.876 INFO:teuthology.orchestra.run.smithi086.stdout:osd.6 smithi192 running (25m) 24s ago 25m 52.8M 3248M 17.2.0 e1d6a67b021e bb7d171b02a1 2024-04-02T05:13:40.876 INFO:teuthology.orchestra.run.smithi086.stdout:osd.7 smithi192 running (25m) 24s ago 25m 54.9M 3248M 17.2.0 e1d6a67b021e 95f5a0353999 2024-04-02T05:13:40.876 INFO:teuthology.orchestra.run.smithi086.stdout:prometheus.a smithi192 *:9095 running (2m) 24s ago 24m 60.8M - 2.43.0 a07b618ecd1d 8378fcdb60e3 2024-04-02T05:13:40.876 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi086.bxzxlp smithi086 *:8000 running (22m) 66s ago 22m 78.0M - 17.2.0 e1d6a67b021e 6725785fbc53 2024-04-02T05:13:40.876 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi192.pmlbot smithi192 *:8000 running (22m) 24s ago 22m 77.7M - 17.2.0 e1d6a67b021e 5ae3638f8b1c 2024-04-02T05:13:41.237 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:40 smithi086 ceph-mon[29505]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:13:41.237 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:40 smithi086 ceph-mon[29505]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:13:41.237 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:40 smithi086 ceph-mon[24811]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:13:41.237 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:40 smithi086 ceph-mon[24811]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:13:41.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:40 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:13:41.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:40 smithi192 ceph-mon[68000]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:13:41.563 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 2'"'"'' 2024-04-02T05:13:42.075 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:41 smithi086 ceph-mon[29505]: from='client.25729 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:13:42.076 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:41 smithi086 ceph-mon[24811]: from='client.25729 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:13:42.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:41 smithi192 ceph-mon[68000]: from='client.25729 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:13:43.031 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:42 smithi086 ceph-mon[29505]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:43.031 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:42 smithi086 ceph-mon[24811]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:43.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:42 smithi192 ceph-mon[68000]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:43.527 INFO:teuthology.orchestra.run.smithi086.stdout:true 2024-04-02T05:13:43.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:13:43 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:13:43] "GET /metrics HTTP/1.1" 200 38001 "" "Prometheus/2.43.0" 2024-04-02T05:13:44.251 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:43 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/978723397' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:44.251 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:43 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/978723397' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:44.279 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch upgrade status' 2024-04-02T05:13:44.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:43 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/978723397' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:45.257 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:44 smithi086 ceph-mon[29505]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:13:45.257 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:44 smithi086 ceph-mon[24811]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:13:45.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:44 smithi192 ceph-mon[68000]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:13:46.254 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:13:46.255 INFO:teuthology.orchestra.run.smithi086.stdout: "target_image": null, 2024-04-02T05:13:46.255 INFO:teuthology.orchestra.run.smithi086.stdout: "in_progress": false, 2024-04-02T05:13:46.255 INFO:teuthology.orchestra.run.smithi086.stdout: "which": "", 2024-04-02T05:13:46.255 INFO:teuthology.orchestra.run.smithi086.stdout: "services_complete": [], 2024-04-02T05:13:46.255 INFO:teuthology.orchestra.run.smithi086.stdout: "progress": null, 2024-04-02T05:13:46.255 INFO:teuthology.orchestra.run.smithi086.stdout: "message": "", 2024-04-02T05:13:46.255 INFO:teuthology.orchestra.run.smithi086.stdout: "is_paused": false 2024-04-02T05:13:46.255 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:13:46.836 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph health detail' 2024-04-02T05:13:47.144 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:46 smithi086 ceph-mon[29505]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:47.144 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:46 smithi086 ceph-mon[24811]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:47.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:46 smithi192 ceph-mon[68000]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:48.250 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:47 smithi086 ceph-mon[24811]: from='client.15852 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:13:48.250 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:47 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:13:48.251 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:47 smithi086 ceph-mon[29505]: from='client.15852 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:13:48.251 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:47 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:13:48.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:47 smithi192 ceph-mon[68000]: from='client.15852 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:13:48.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:47 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:13:48.859 INFO:teuthology.orchestra.run.smithi086.stdout:HEALTH_OK 2024-04-02T05:13:49.219 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:48 smithi086 ceph-mon[24811]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:49.219 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:48 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/427897759' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T05:13:49.219 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:48 smithi086 ceph-mon[29505]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:49.220 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:48 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/427897759' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T05:13:49.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:48 smithi192 ceph-mon[68000]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:49.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:48 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/427897759' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T05:13:49.669 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '"'"'{print $2}'"'"')' 2024-04-02T05:13:51.212 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:50 smithi086 ceph-mon[24811]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:13:51.212 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:50 smithi086 ceph-mon[24811]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:13:51.212 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:50 smithi086 ceph-mon[29505]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:13:51.212 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:50 smithi086 ceph-mon[29505]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:13:51.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:50 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:13:51.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:50 smithi192 ceph-mon[68000]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:13:53.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:52 smithi086 ceph-mon[24811]: from='client.15864 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:13:53.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:52 smithi086 ceph-mon[24811]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:53.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:52 smithi086 ceph-mon[24811]: from='client.15870 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "daemon_types": "mon", "hosts": "smithi086", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:13:53.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:52 smithi086 ceph-mon[29505]: from='client.15864 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:13:53.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:52 smithi086 ceph-mon[29505]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:53.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:52 smithi086 ceph-mon[29505]: from='client.15870 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "daemon_types": "mon", "hosts": "smithi086", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:13:53.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:52 smithi192 ceph-mon[68000]: from='client.15864 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:13:53.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:52 smithi192 ceph-mon[68000]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:53.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:52 smithi192 ceph-mon[68000]: from='client.15870 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "daemon_types": "mon", "hosts": "smithi086", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:13:53.552 INFO:teuthology.orchestra.run.smithi086.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:13:53.832 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:13:53 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:13:53] "GET /metrics HTTP/1.1" 200 37999 "" "Prometheus/2.43.0" 2024-04-02T05:13:54.210 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-02T05:13:54.745 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:54 smithi086 ceph-mon[29505]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:13:54.745 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:54 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:54.745 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:54 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:13:54.745 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:54 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:13:54.746 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:54 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:54.746 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:54 smithi086 ceph-mon[29505]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:13:54.746 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:54 smithi086 ceph-mon[29505]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:13:54.746 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:54 smithi086 ceph-mon[24811]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:13:54.746 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:54 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:54.746 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:54 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:13:54.746 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:54 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:13:54.746 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:54 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:54.746 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:54 smithi086 ceph-mon[24811]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:13:54.746 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:54 smithi086 ceph-mon[24811]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:13:54.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:54 smithi192 ceph-mon[68000]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:13:54.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:54 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:54.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:54 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:13:54.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:54 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:13:54.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:54 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:54.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:54 smithi192 ceph-mon[68000]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:13:54.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:54 smithi192 ceph-mon[68000]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:13:56.712 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:56 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:56.713 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:56 smithi086 ceph-mon[24811]: Upgrade: Target is version 19.0.0-1979-g6cec793d (squid) 2024-04-02T05:13:56.713 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:56 smithi086 ceph-mon[24811]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7bd5b84926d4a0d6c51e5562e965bff211cec2740c9f4dec9b5e00a2bbb75eae'] 2024-04-02T05:13:56.713 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:56 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:13:56.713 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:56 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:56.713 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:56 smithi086 ceph-mon[24811]: Upgrade: Setting container_image for all mgr 2024-04-02T05:13:56.713 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:56 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:56.713 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:56 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-02T05:13:56.713 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:56 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-02T05:13:56.713 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:56 smithi086 ceph-mon[24811]: Upgrade: It appears safe to stop mon.a 2024-04-02T05:13:56.713 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:56 smithi086 ceph-mon[24811]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:56.713 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:56 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:56.714 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:56 smithi086 ceph-mon[29505]: Upgrade: Target is version 19.0.0-1979-g6cec793d (squid) 2024-04-02T05:13:56.714 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:56 smithi086 ceph-mon[29505]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7bd5b84926d4a0d6c51e5562e965bff211cec2740c9f4dec9b5e00a2bbb75eae'] 2024-04-02T05:13:56.714 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:56 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:13:56.714 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:56 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:56.714 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:56 smithi086 ceph-mon[29505]: Upgrade: Setting container_image for all mgr 2024-04-02T05:13:56.714 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:56 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:56.714 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:56 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-02T05:13:56.714 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:56 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-02T05:13:56.714 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:56 smithi086 ceph-mon[29505]: Upgrade: It appears safe to stop mon.a 2024-04-02T05:13:56.714 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:56 smithi086 ceph-mon[29505]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:56.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:56 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:56.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:56 smithi192 ceph-mon[68000]: Upgrade: Target is version 19.0.0-1979-g6cec793d (squid) 2024-04-02T05:13:56.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:56 smithi192 ceph-mon[68000]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7bd5b84926d4a0d6c51e5562e965bff211cec2740c9f4dec9b5e00a2bbb75eae'] 2024-04-02T05:13:56.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:56 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:13:56.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:56 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:13:56.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:56 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all mgr 2024-04-02T05:13:56.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:56 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:56.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:56 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-02T05:13:56.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:56 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-02T05:13:56.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:56 smithi192 ceph-mon[68000]: Upgrade: It appears safe to stop mon.a 2024-04-02T05:13:56.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:56 smithi192 ceph-mon[68000]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:58.097 INFO:teuthology.orchestra.run.smithi086.stdout:true 2024-04-02T05:13:58.605 INFO:teuthology.orchestra.run.smithi086.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T05:13:58.605 INFO:teuthology.orchestra.run.smithi086.stdout:alertmanager.a smithi086 *:9093,9094 running (2m) 84s ago 23m 16.0M - 0.25.0 c8568f914cd2 660ea62bb6b9 2024-04-02T05:13:58.606 INFO:teuthology.orchestra.run.smithi086.stdout:grafana.a smithi192 *:3000 running (105s) 41s ago 23m 74.7M - 9.4.12 2bacad6d85d8 259af9e63aa5 2024-04-02T05:13:58.606 INFO:teuthology.orchestra.run.smithi086.stdout:iscsi.foo.smithi086.rsbqwx smithi086 running (89s) 84s ago 23m 52.0M - 3.5 e1d6a67b021e fe728eb940f7 2024-04-02T05:13:58.606 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.x smithi192 *:8443,9283,8765 running (3m) 41s ago 28m 450M - 19.0.0-1979-g6cec793d 78f830190839 dd1e6d0f0a95 2024-04-02T05:13:58.606 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.y smithi086 *:8443,9283,8765 running (12m) 84s ago 30m 510M - 19.0.0-1979-g6cec793d 78f830190839 3dca54a018ab 2024-04-02T05:13:58.606 INFO:teuthology.orchestra.run.smithi086.stdout:mon.a smithi086 running (30m) 84s ago 30m 79.8M 2048M 17.2.0 e1d6a67b021e b6c507ab96f8 2024-04-02T05:13:58.606 INFO:teuthology.orchestra.run.smithi086.stdout:mon.b smithi192 running (46s) 41s ago 28m 18.6M 2048M 19.0.0-1979-g6cec793d 78f830190839 4125e997188f 2024-04-02T05:13:58.606 INFO:teuthology.orchestra.run.smithi086.stdout:mon.c smithi086 running (28m) 84s ago 28m 58.3M 2048M 17.2.0 e1d6a67b021e 49275483a2d4 2024-04-02T05:13:58.606 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.a smithi086 *:9100 running (2m) 84s ago 24m 15.8M - 1.5.0 0da6a335fe13 05a844569d7b 2024-04-02T05:13:58.606 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.b smithi192 *:9100 running (2m) 41s ago 24m 17.0M - 1.5.0 0da6a335fe13 1a7d6788d58b 2024-04-02T05:13:58.606 INFO:teuthology.orchestra.run.smithi086.stdout:osd.0 smithi086 running (28m) 84s ago 28m 57.5M 2992M 17.2.0 e1d6a67b021e 6d90c60d1c71 2024-04-02T05:13:58.607 INFO:teuthology.orchestra.run.smithi086.stdout:osd.1 smithi086 running (27m) 84s ago 27m 56.5M 2992M 17.2.0 e1d6a67b021e 203689f71603 2024-04-02T05:13:58.607 INFO:teuthology.orchestra.run.smithi086.stdout:osd.2 smithi086 running (27m) 84s ago 27m 50.4M 2992M 17.2.0 e1d6a67b021e 472c860a9af8 2024-04-02T05:13:58.607 INFO:teuthology.orchestra.run.smithi086.stdout:osd.3 smithi086 running (26m) 84s ago 26m 52.7M 2992M 17.2.0 e1d6a67b021e 327538dac44e 2024-04-02T05:13:58.607 INFO:teuthology.orchestra.run.smithi086.stdout:osd.4 smithi192 running (26m) 41s ago 26m 56.0M 3248M 17.2.0 e1d6a67b021e 6535d7919a2d 2024-04-02T05:13:58.607 INFO:teuthology.orchestra.run.smithi086.stdout:osd.5 smithi192 running (26m) 41s ago 26m 54.9M 3248M 17.2.0 e1d6a67b021e 7259249605f2 2024-04-02T05:13:58.607 INFO:teuthology.orchestra.run.smithi086.stdout:osd.6 smithi192 running (25m) 41s ago 25m 52.8M 3248M 17.2.0 e1d6a67b021e bb7d171b02a1 2024-04-02T05:13:58.607 INFO:teuthology.orchestra.run.smithi086.stdout:osd.7 smithi192 running (25m) 41s ago 25m 54.9M 3248M 17.2.0 e1d6a67b021e 95f5a0353999 2024-04-02T05:13:58.607 INFO:teuthology.orchestra.run.smithi086.stdout:prometheus.a smithi192 *:9095 running (2m) 41s ago 24m 60.8M - 2.43.0 a07b618ecd1d 8378fcdb60e3 2024-04-02T05:13:58.607 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi086.bxzxlp smithi086 *:8000 running (23m) 84s ago 23m 78.0M - 17.2.0 e1d6a67b021e 6725785fbc53 2024-04-02T05:13:58.607 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi192.pmlbot smithi192 *:8000 running (23m) 41s ago 23m 77.7M - 17.2.0 e1d6a67b021e 5ae3638f8b1c 2024-04-02T05:13:59.017 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:13:59.017 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": { 2024-04-02T05:13:59.017 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-04-02T05:13:59.017 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 1 2024-04-02T05:13:59.017 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:13:59.017 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": { 2024-04-02T05:13:59.017 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 2 2024-04-02T05:13:59.017 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:13:59.018 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": { 2024-04-02T05:13:59.018 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-02T05:13:59.018 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:13:59.018 INFO:teuthology.orchestra.run.smithi086.stdout: "mds": {}, 2024-04-02T05:13:59.018 INFO:teuthology.orchestra.run.smithi086.stdout: "rgw": { 2024-04-02T05:13:59.018 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-02T05:13:59.018 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:13:59.018 INFO:teuthology.orchestra.run.smithi086.stdout: "overall": { 2024-04-02T05:13:59.018 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-04-02T05:13:59.018 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 3 2024-04-02T05:13:59.018 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-04-02T05:13:59.018 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:13:59.279 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:58 smithi086 ceph-mon[29505]: from='client.15876 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:13:59.280 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:58 smithi086 ceph-mon[29505]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:59.280 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:58 smithi086 ceph-mon[29505]: Upgrade: Updating mon.a 2024-04-02T05:13:59.280 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:58 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:59.280 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:58 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T05:13:59.280 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:58 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T05:13:59.280 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:58 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:13:59.280 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:58 smithi086 ceph-mon[29505]: Deploying daemon mon.a on smithi086 2024-04-02T05:13:59.280 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:58 smithi086 ceph-mon[24811]: from='client.15876 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:13:59.280 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:58 smithi086 ceph-mon[24811]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:59.280 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:58 smithi086 ceph-mon[24811]: Upgrade: Updating mon.a 2024-04-02T05:13:59.280 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:58 smithi086 ceph-mon[24811]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:59.281 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:58 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T05:13:59.281 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:58 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T05:13:59.281 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:58 smithi086 ceph-mon[24811]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:13:59.281 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:58 smithi086 ceph-mon[24811]: Deploying daemon mon.a on smithi086 2024-04-02T05:13:59.348 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:13:59.348 INFO:teuthology.orchestra.run.smithi086.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", 2024-04-02T05:13:59.348 INFO:teuthology.orchestra.run.smithi086.stdout: "in_progress": true, 2024-04-02T05:13:59.349 INFO:teuthology.orchestra.run.smithi086.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi086", 2024-04-02T05:13:59.349 INFO:teuthology.orchestra.run.smithi086.stdout: "services_complete": [], 2024-04-02T05:13:59.349 INFO:teuthology.orchestra.run.smithi086.stdout: "progress": "0/2 daemons upgraded", 2024-04-02T05:13:59.349 INFO:teuthology.orchestra.run.smithi086.stdout: "message": "Currently upgrading mon daemons", 2024-04-02T05:13:59.349 INFO:teuthology.orchestra.run.smithi086.stdout: "is_paused": false 2024-04-02T05:13:59.349 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:13:59.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:58 smithi192 ceph-mon[68000]: from='client.15876 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:13:59.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:58 smithi192 ceph-mon[68000]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:13:59.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:58 smithi192 ceph-mon[68000]: Upgrade: Updating mon.a 2024-04-02T05:13:59.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:58 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='mgr.y' 2024-04-02T05:13:59.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:58 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T05:13:59.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:58 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T05:13:59.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:58 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:13:59.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:58 smithi192 ceph-mon[68000]: Deploying daemon mon.a on smithi086 2024-04-02T05:14:00.118 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:59 smithi086 ceph-mon[29505]: from='client.15882 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:14:00.118 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:59 smithi086 ceph-mon[29505]: from='client.25777 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:14:00.118 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:13:59 smithi086 ceph-mon[29505]: from='client.? 172.21.15.86:0/1019768394' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:14:00.119 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:59 smithi086 systemd[1]: Stopping Ceph mon.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:14:00.119 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:59 smithi086 ceph-mon[24811]: from='client.15882 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:14:00.119 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:59 smithi086 ceph-mon[24811]: from='client.25777 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:14:00.119 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:13:59 smithi086 ceph-mon[24811]: from='client.? 172.21.15.86:0/1019768394' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:14:00.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:59 smithi192 ceph-mon[68000]: from='client.15882 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:14:00.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:59 smithi192 ceph-mon[68000]: from='client.25777 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:14:00.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:13:59 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/1019768394' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:14:00.526 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:00 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-a[24807]: 2024-04-02T05:14:00.115+0000 7f42b8087700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T05:14:00.526 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:00 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-a[24807]: 2024-04-02T05:14:00.115+0000 7f42b8087700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-04-02T05:14:00.527 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:00 smithi086 podman[86280]: 2024-04-02 05:14:00.186965386 +0000 UTC m=+0.197723930 container died b6c507ab96f87cb05b52f8388f6b58a90d5c1eee4c4cf01f4b541161b2096dee (image=quay.io/ceph/ceph:v17.2.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-a, name=centos-stream, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., io.openshift.expose-services=, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, RELEASE=HEAD, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-02T05:14:00.832 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:00 smithi086 podman[86280]: 2024-04-02 05:14:00.632144879 +0000 UTC m=+0.642903455 container cleanup b6c507ab96f87cb05b52f8388f6b58a90d5c1eee4c4cf01f4b541161b2096dee (image=quay.io/ceph/ceph:v17.2.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-a, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, GIT_BRANCH=HEAD, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, architecture=x86_64, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, RELEASE=HEAD, distribution-scope=public, io.buildah.version=1.19.8) 2024-04-02T05:14:00.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:00 smithi086 bash[86280]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-a 2024-04-02T05:14:01.307 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:00 smithi086 podman[86294]: 2024-04-02 05:14:00.941730992 +0000 UTC m=+0.753096133 container remove b6c507ab96f87cb05b52f8388f6b58a90d5c1eee4c4cf01f4b541161b2096dee (image=quay.io/ceph/ceph:v17.2.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-a, release=754, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, name=centos-stream, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, RELEASE=HEAD, com.redhat.component=centos-stream-container, vcs-type=git, GIT_BRANCH=HEAD) 2024-04-02T05:14:01.307 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mon.a.service: Deactivated successfully. 2024-04-02T05:14:01.307 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 systemd[1]: Stopped Ceph mon.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:14:01.307 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mon.a.service: Consumed 25.651s CPU time. 2024-04-02T05:14:01.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 systemd[1]: Starting Ceph mon.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:14:01.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 podman[86424]: 2024-04-02 05:14:01.422364838 +0000 UTC m=+0.017783022 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:14:01.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 podman[86424]: 2024-04-02 05:14:01.534460278 +0000 UTC m=+0.129878493 container create 8e478dd4da2dbd4ceb762aa4372bed999d44e3977f4dd14fd300b2ada2762906 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-a, RELEASE=squid-6cec793, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-04-02T05:14:02.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 podman[86424]: 2024-04-02 05:14:01.787564851 +0000 UTC m=+0.382983026 container init 8e478dd4da2dbd4ceb762aa4372bed999d44e3977f4dd14fd300b2ada2762906 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-a, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-6cec793, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-04-02T05:14:02.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 podman[86424]: 2024-04-02 05:14:01.790763595 +0000 UTC m=+0.386181771 container start 8e478dd4da2dbd4ceb762aa4372bed999d44e3977f4dd14fd300b2ada2762906 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-a, org.label-schema.license=GPLv2, RELEASE=squid-6cec793, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5) 2024-04-02T05:14:02.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: set uid:gid to 167:167 (ceph:ceph) 2024-04-02T05:14:02.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev), process ceph-mon, pid 2 2024-04-02T05:14:02.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: pidfile_write: ignore empty --pid-file 2024-04-02T05:14:02.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: load: jerasure load: lrc 2024-04-02T05:14:02.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: RocksDB version: 7.9.2 2024-04-02T05:14:02.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Git sha 0 2024-04-02T05:14:02.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Compile date 2024-04-01 15:23:26 2024-04-02T05:14:02.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: DB SUMMARY 2024-04-02T05:14:02.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: DB Session ID: RAVC1ZW2LZRY0MSX9TV2 2024-04-02T05:14:02.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: CURRENT file: CURRENT 2024-04-02T05:14:02.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: IDENTITY file: IDENTITY 2024-04-02T05:14:02.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: MANIFEST file: MANIFEST-000015 size: 2889 Bytes 2024-04-02T05:14:02.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000060.sst 2024-04-02T05:14:02.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000058.log size: 2612181 ; 2024-04-02T05:14:02.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.error_if_exists: 0 2024-04-02T05:14:02.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.create_if_missing: 0 2024-04-02T05:14:02.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.paranoid_checks: 1 2024-04-02T05:14:02.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-02T05:14:02.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-02T05:14:02.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-02T05:14:02.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.env: 0x560903644c60 2024-04-02T05:14:02.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.fs: PosixFileSystem 2024-04-02T05:14:02.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.info_log: 0x560904707ae0 2024-04-02T05:14:02.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.max_file_opening_threads: 16 2024-04-02T05:14:02.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.statistics: (nil) 2024-04-02T05:14:02.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.use_fsync: 0 2024-04-02T05:14:02.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.max_log_file_size: 0 2024-04-02T05:14:02.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-02T05:14:02.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-02T05:14:02.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.keep_log_file_num: 1000 2024-04-02T05:14:02.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.recycle_log_file_num: 0 2024-04-02T05:14:02.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.allow_fallocate: 1 2024-04-02T05:14:02.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.allow_mmap_reads: 0 2024-04-02T05:14:02.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.allow_mmap_writes: 0 2024-04-02T05:14:02.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.use_direct_reads: 0 2024-04-02T05:14:02.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-02T05:14:02.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.create_missing_column_families: 0 2024-04-02T05:14:02.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.db_log_dir: 2024-04-02T05:14:02.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.wal_dir: 2024-04-02T05:14:02.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-02T05:14:02.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-02T05:14:02.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-02T05:14:02.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-02T05:14:02.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-02T05:14:02.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-02T05:14:02.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.advise_random_on_open: 1 2024-04-02T05:14:02.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.db_write_buffer_size: 0 2024-04-02T05:14:02.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.write_buffer_manager: 0x5609046da820 2024-04-02T05:14:02.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-02T05:14:02.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-02T05:14:02.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-02T05:14:02.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.rate_limiter: (nil) 2024-04-02T05:14:02.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-02T05:14:02.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.wal_recovery_mode: 2 2024-04-02T05:14:02.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.enable_thread_tracking: 0 2024-04-02T05:14:02.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.enable_pipelined_write: 0 2024-04-02T05:14:02.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.unordered_write: 0 2024-04-02T05:14:02.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-02T05:14:02.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-02T05:14:02.088 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-02T05:14:02.088 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-02T05:14:02.088 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.row_cache: None 2024-04-02T05:14:02.088 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.wal_filter: None 2024-04-02T05:14:02.088 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-02T05:14:02.089 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.allow_ingest_behind: 0 2024-04-02T05:14:02.089 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.two_write_queues: 0 2024-04-02T05:14:02.089 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.manual_wal_flush: 0 2024-04-02T05:14:02.089 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.wal_compression: 0 2024-04-02T05:14:02.089 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.atomic_flush: 0 2024-04-02T05:14:02.089 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-02T05:14:02.089 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-02T05:14:02.089 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-02T05:14:02.090 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.log_readahead_size: 0 2024-04-02T05:14:02.090 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-02T05:14:02.090 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.best_efforts_recovery: 0 2024-04-02T05:14:02.090 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-02T05:14:02.090 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-02T05:14:02.090 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.allow_data_in_errors: 0 2024-04-02T05:14:02.090 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.db_host_id: __hostname__ 2024-04-02T05:14:02.090 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-02T05:14:02.090 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.max_background_jobs: 2 2024-04-02T05:14:02.090 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.max_background_compactions: -1 2024-04-02T05:14:02.090 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.max_subcompactions: 1 2024-04-02T05:14:02.090 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-02T05:14:02.090 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-02T05:14:02.090 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-02T05:14:02.091 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.max_total_wal_size: 0 2024-04-02T05:14:02.091 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-02T05:14:02.091 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-02T05:14:02.091 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-02T05:14:02.091 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-02T05:14:02.091 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.max_open_files: -1 2024-04-02T05:14:02.091 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.bytes_per_sync: 0 2024-04-02T05:14:02.091 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-02T05:14:02.091 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-02T05:14:02.091 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.compaction_readahead_size: 0 2024-04-02T05:14:02.091 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.max_background_flushes: -1 2024-04-02T05:14:02.091 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Compression algorithms supported: 2024-04-02T05:14:02.091 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-02T05:14:02.091 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: kZSTD supported: 0 2024-04-02T05:14:02.091 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: kXpressCompression supported: 0 2024-04-02T05:14:02.092 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: kLZ4HCCompression supported: 1 2024-04-02T05:14:02.092 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: kZlibCompression supported: 1 2024-04-02T05:14:02.092 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: kSnappyCompression supported: 1 2024-04-02T05:14:02.092 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: kLZ4Compression supported: 1 2024-04-02T05:14:02.092 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: kBZip2Compression supported: 0 2024-04-02T05:14:02.092 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-02T05:14:02.092 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-02T05:14:02.092 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-04-02T05:14:02.093 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-02T05:14:02.093 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-02T05:14:02.093 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.merge_operator: 2024-04-02T05:14:02.093 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.compaction_filter: None 2024-04-02T05:14:02.093 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.compaction_filter_factory: None 2024-04-02T05:14:02.093 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.sst_partitioner_factory: None 2024-04-02T05:14:02.093 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-02T05:14:02.093 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-02T05:14:02.093 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x560904707be0) 2024-04-02T05:14:02.093 INFO:journalctl@ceph.mon.a.smithi086.stdout: cache_index_and_filter_blocks: 1 2024-04-02T05:14:02.093 INFO:journalctl@ceph.mon.a.smithi086.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-02T05:14:02.093 INFO:journalctl@ceph.mon.a.smithi086.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-02T05:14:02.093 INFO:journalctl@ceph.mon.a.smithi086.stdout: pin_top_level_index_and_filter: 1 2024-04-02T05:14:02.093 INFO:journalctl@ceph.mon.a.smithi086.stdout: index_type: 0 2024-04-02T05:14:02.093 INFO:journalctl@ceph.mon.a.smithi086.stdout: data_block_index_type: 0 2024-04-02T05:14:02.094 INFO:journalctl@ceph.mon.a.smithi086.stdout: index_shortening: 1 2024-04-02T05:14:02.094 INFO:journalctl@ceph.mon.a.smithi086.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-02T05:14:02.094 INFO:journalctl@ceph.mon.a.smithi086.stdout: checksum: 4 2024-04-02T05:14:02.094 INFO:journalctl@ceph.mon.a.smithi086.stdout: no_block_cache: 0 2024-04-02T05:14:02.094 INFO:journalctl@ceph.mon.a.smithi086.stdout: block_cache: 0x56090470b090 2024-04-02T05:14:02.094 INFO:journalctl@ceph.mon.a.smithi086.stdout: block_cache_name: BinnedLRUCache 2024-04-02T05:14:02.094 INFO:journalctl@ceph.mon.a.smithi086.stdout: block_cache_options: 2024-04-02T05:14:02.094 INFO:journalctl@ceph.mon.a.smithi086.stdout: capacity : 536870912 2024-04-02T05:14:02.094 INFO:journalctl@ceph.mon.a.smithi086.stdout: num_shard_bits : 4 2024-04-02T05:14:02.094 INFO:journalctl@ceph.mon.a.smithi086.stdout: strict_capacity_limit : 0 2024-04-02T05:14:02.094 INFO:journalctl@ceph.mon.a.smithi086.stdout: high_pri_pool_ratio: 0.000 2024-04-02T05:14:02.094 INFO:journalctl@ceph.mon.a.smithi086.stdout: block_cache_compressed: (nil) 2024-04-02T05:14:02.094 INFO:journalctl@ceph.mon.a.smithi086.stdout: persistent_cache: (nil) 2024-04-02T05:14:02.094 INFO:journalctl@ceph.mon.a.smithi086.stdout: block_size: 4096 2024-04-02T05:14:02.094 INFO:journalctl@ceph.mon.a.smithi086.stdout: block_size_deviation: 10 2024-04-02T05:14:02.095 INFO:journalctl@ceph.mon.a.smithi086.stdout: block_restart_interval: 16 2024-04-02T05:14:02.095 INFO:journalctl@ceph.mon.a.smithi086.stdout: index_block_restart_interval: 1 2024-04-02T05:14:02.095 INFO:journalctl@ceph.mon.a.smithi086.stdout: metadata_block_size: 4096 2024-04-02T05:14:02.095 INFO:journalctl@ceph.mon.a.smithi086.stdout: partition_filters: 0 2024-04-02T05:14:02.095 INFO:journalctl@ceph.mon.a.smithi086.stdout: use_delta_encoding: 1 2024-04-02T05:14:02.095 INFO:journalctl@ceph.mon.a.smithi086.stdout: filter_policy: bloomfilter 2024-04-02T05:14:02.095 INFO:journalctl@ceph.mon.a.smithi086.stdout: whole_key_filtering: 1 2024-04-02T05:14:02.095 INFO:journalctl@ceph.mon.a.smithi086.stdout: verify_compression: 0 2024-04-02T05:14:02.095 INFO:journalctl@ceph.mon.a.smithi086.stdout: read_amp_bytes_per_bit: 0 2024-04-02T05:14:02.095 INFO:journalctl@ceph.mon.a.smithi086.stdout: format_version: 5 2024-04-02T05:14:02.095 INFO:journalctl@ceph.mon.a.smithi086.stdout: enable_index_compression: 1 2024-04-02T05:14:02.095 INFO:journalctl@ceph.mon.a.smithi086.stdout: block_align: 0 2024-04-02T05:14:02.095 INFO:journalctl@ceph.mon.a.smithi086.stdout: max_auto_readahead_size: 262144 2024-04-02T05:14:02.095 INFO:journalctl@ceph.mon.a.smithi086.stdout: prepopulate_block_cache: 0 2024-04-02T05:14:02.095 INFO:journalctl@ceph.mon.a.smithi086.stdout: initial_auto_readahead_size: 8192 2024-04-02T05:14:02.096 INFO:journalctl@ceph.mon.a.smithi086.stdout: num_file_reads_for_auto_readahead: 2 2024-04-02T05:14:02.096 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.write_buffer_size: 33554432 2024-04-02T05:14:02.096 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.max_write_buffer_number: 2 2024-04-02T05:14:02.096 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.compression: NoCompression 2024-04-02T05:14:02.096 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.bottommost_compression: Disabled 2024-04-02T05:14:02.096 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.prefix_extractor: nullptr 2024-04-02T05:14:02.096 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-02T05:14:02.096 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.num_levels: 7 2024-04-02T05:14:02.096 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-02T05:14:02.096 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-02T05:14:02.096 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-02T05:14:02.096 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-02T05:14:02.097 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-02T05:14:02.097 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-02T05:14:02.097 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-02T05:14:02.097 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-02T05:14:02.097 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-02T05:14:02.097 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-02T05:14:02.097 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-02T05:14:02.097 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-02T05:14:02.097 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-02T05:14:02.097 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.compression_opts.level: 32767 2024-04-02T05:14:02.097 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.compression_opts.strategy: 0 2024-04-02T05:14:02.097 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-02T05:14:02.097 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-02T05:14:02.097 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-02T05:14:02.097 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-02T05:14:02.098 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.compression_opts.enabled: false 2024-04-02T05:14:02.098 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-02T05:14:02.098 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-02T05:14:02.098 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-02T05:14:02.098 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-02T05:14:02.098 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.target_file_size_base: 67108864 2024-04-02T05:14:02.098 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-02T05:14:02.098 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-02T05:14:02.098 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-02T05:14:02.098 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-02T05:14:02.098 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-02T05:14:02.098 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-02T05:14:02.098 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-02T05:14:02.098 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-02T05:14:02.099 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-02T05:14:02.099 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-02T05:14:02.099 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-02T05:14:02.099 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-02T05:14:02.099 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-02T05:14:02.099 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-02T05:14:02.099 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.arena_block_size: 1048576 2024-04-02T05:14:02.099 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-02T05:14:02.099 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-02T05:14:02.099 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.disable_auto_compactions: 0 2024-04-02T05:14:02.099 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-02T05:14:02.099 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-02T05:14:02.099 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-02T05:14:02.099 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-02T05:14:02.099 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-02T05:14:02.100 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-02T05:14:02.100 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-02T05:14:02.100 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-02T05:14:02.100 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-02T05:14:02.100 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-02T05:14:02.100 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.table_properties_collectors: 2024-04-02T05:14:02.100 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.inplace_update_support: 0 2024-04-02T05:14:02.100 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-02T05:14:02.100 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-02T05:14:02.100 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-02T05:14:02.100 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-02T05:14:02.101 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.bloom_locality: 0 2024-04-02T05:14:02.101 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.max_successive_merges: 0 2024-04-02T05:14:02.101 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-02T05:14:02.101 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.paranoid_file_checks: 0 2024-04-02T05:14:02.101 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.force_consistency_checks: 1 2024-04-02T05:14:02.101 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.report_bg_io_stats: 0 2024-04-02T05:14:02.101 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.ttl: 2592000 2024-04-02T05:14:02.101 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-02T05:14:02.101 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-02T05:14:02.101 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-02T05:14:02.101 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.enable_blob_files: false 2024-04-02T05:14:02.101 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.min_blob_size: 0 2024-04-02T05:14:02.101 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.blob_file_size: 268435456 2024-04-02T05:14:02.101 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-02T05:14:02.102 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-02T05:14:02.102 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-02T05:14:02.102 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-02T05:14:02.102 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-02T05:14:02.102 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.blob_file_starting_level: 0 2024-04-02T05:14:02.103 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-02T05:14:02.103 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 60.sst 2024-04-02T05:14:02.103 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-02T05:14:02.103 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 62, last_sequence is 26765, log_number is 58,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-02T05:14:02.103 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 58 2024-04-02T05:14:02.103 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 209ac2fa-d72e-482f-8f50-19aca96a133c 2024-04-02T05:14:02.103 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712034841821134, "job": 1, "event": "recovery_started", "wal_files": [58]} 2024-04-02T05:14:02.103 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #58 mode 2 2024-04-02T05:14:02.103 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712034841831562, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 63, "file_size": 2397361, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 26766, "largest_seqno": 29032, "table_properties": {"data_size": 2388373, "index_size": 5408, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 2565, "raw_key_size": 24753, "raw_average_key_size": 24, "raw_value_size": 2367938, "raw_average_value_size": 2365, "num_data_blocks": 241, "num_entries": 1001, "num_filter_entries": 1001, "num_deletions": 8, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1712034841, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "209ac2fa-d72e-482f-8f50-19aca96a133c", "db_session_id": "RAVC1ZW2LZRY0MSX9TV2", "orig_file_number": 63, "seqno_to_time_mapping": "N/A"}} 2024-04-02T05:14:02.103 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712034841831673, "job": 1, "event": "recovery_finished"} 2024-04-02T05:14:02.103 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: [db/version_set.cc:5047] Creating manifest 65 2024-04-02T05:14:02.103 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-02T05:14:02.103 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000058.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-02T05:14:02.103 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5609047fc000 2024-04-02T05:14:02.104 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: DB pointer 0x5609047e6000 2024-04-02T05:14:02.104 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-02T05:14:02.104 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-02T05:14:02.104 INFO:journalctl@ceph.mon.a.smithi086.stdout: ** DB Stats ** 2024-04-02T05:14:02.104 INFO:journalctl@ceph.mon.a.smithi086.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-02T05:14:02.104 INFO:journalctl@ceph.mon.a.smithi086.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-02T05:14:02.104 INFO:journalctl@ceph.mon.a.smithi086.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-02T05:14:02.104 INFO:journalctl@ceph.mon.a.smithi086.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-02T05:14:02.104 INFO:journalctl@ceph.mon.a.smithi086.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-02T05:14:02.104 INFO:journalctl@ceph.mon.a.smithi086.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-02T05:14:02.104 INFO:journalctl@ceph.mon.a.smithi086.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-02T05:14:02.104 INFO:journalctl@ceph.mon.a.smithi086.stdout: 2024-04-02T05:14:02.104 INFO:journalctl@ceph.mon.a.smithi086.stdout: ** Compaction Stats [default] ** 2024-04-02T05:14:02.104 INFO:journalctl@ceph.mon.a.smithi086.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-02T05:14:02.105 INFO:journalctl@ceph.mon.a.smithi086.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-02T05:14:02.105 INFO:journalctl@ceph.mon.a.smithi086.stdout: L0 1/0 2.29 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 590.2 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-02T05:14:02.105 INFO:journalctl@ceph.mon.a.smithi086.stdout: L6 1/0 9.10 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-02T05:14:02.105 INFO:journalctl@ceph.mon.a.smithi086.stdout: Sum 2/0 11.39 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 590.2 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-02T05:14:02.105 INFO:journalctl@ceph.mon.a.smithi086.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 590.2 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-02T05:14:02.105 INFO:journalctl@ceph.mon.a.smithi086.stdout: 2024-04-02T05:14:02.105 INFO:journalctl@ceph.mon.a.smithi086.stdout: ** Compaction Stats [default] ** 2024-04-02T05:14:02.105 INFO:journalctl@ceph.mon.a.smithi086.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-02T05:14:02.105 INFO:journalctl@ceph.mon.a.smithi086.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T05:14:02.105 INFO:journalctl@ceph.mon.a.smithi086.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 590.2 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-02T05:14:02.105 INFO:journalctl@ceph.mon.a.smithi086.stdout: 2024-04-02T05:14:02.105 INFO:journalctl@ceph.mon.a.smithi086.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-02T05:14:02.105 INFO:journalctl@ceph.mon.a.smithi086.stdout: 2024-04-02T05:14:02.105 INFO:journalctl@ceph.mon.a.smithi086.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-02T05:14:02.105 INFO:journalctl@ceph.mon.a.smithi086.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-04-02T05:14:02.106 INFO:journalctl@ceph.mon.a.smithi086.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-02T05:14:02.106 INFO:journalctl@ceph.mon.a.smithi086.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-02T05:14:02.106 INFO:journalctl@ceph.mon.a.smithi086.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-02T05:14:02.106 INFO:journalctl@ceph.mon.a.smithi086.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-02T05:14:02.106 INFO:journalctl@ceph.mon.a.smithi086.stdout: Cumulative compaction: 0.00 GB write, 154.69 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T05:14:02.106 INFO:journalctl@ceph.mon.a.smithi086.stdout: Interval compaction: 0.00 GB write, 154.69 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T05:14:02.106 INFO:journalctl@ceph.mon.a.smithi086.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-02T05:14:02.106 INFO:journalctl@ceph.mon.a.smithi086.stdout: Block cache BinnedLRUCache@0x56090470b090#2 capacity: 512.00 MB usage: 8.86 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-04-02T05:14:02.106 INFO:journalctl@ceph.mon.a.smithi086.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,2.78 KB,0.000530481%) IndexBlock(1,6.08 KB,0.00115931%) Misc(1,0.00 KB,0%) 2024-04-02T05:14:02.106 INFO:journalctl@ceph.mon.a.smithi086.stdout: 2024-04-02T05:14:02.106 INFO:journalctl@ceph.mon.a.smithi086.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-02T05:14:02.106 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: starting mon.a rank 0 at public addrs [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] at bind addrs [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 2024-04-02T05:14:02.106 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: mon.a@-1(???) e3 preinit fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 2024-04-02T05:14:02.106 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: mon.a@-1(???).mds e1 new map 2024-04-02T05:14:02.107 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: mon.a@-1(???).mds e1 print_map 2024-04-02T05:14:02.107 INFO:journalctl@ceph.mon.a.smithi086.stdout: e1 2024-04-02T05:14:02.107 INFO:journalctl@ceph.mon.a.smithi086.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-02T05:14:02.107 INFO:journalctl@ceph.mon.a.smithi086.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-02T05:14:02.107 INFO:journalctl@ceph.mon.a.smithi086.stdout: legacy client fscid: -1 2024-04-02T05:14:02.107 INFO:journalctl@ceph.mon.a.smithi086.stdout: 2024-04-02T05:14:02.107 INFO:journalctl@ceph.mon.a.smithi086.stdout: No filesystems configured 2024-04-02T05:14:02.107 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: mon.a@-1(???).osd e110 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-02T05:14:02.107 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: mon.a@-1(???).osd e110 crush map has features 432629239337189376, adjusting msgr requires 2024-04-02T05:14:02.107 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: mon.a@-1(???).osd e110 crush map has features 432629239337189376, adjusting msgr requires 2024-04-02T05:14:02.107 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: mon.a@-1(???).osd e110 crush map has features 432629239337189376, adjusting msgr requires 2024-04-02T05:14:02.107 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 ceph-mon[86443]: mon.a@-1(???).paxosservice(auth 1..27) refresh upgraded, format 0 -> 3 2024-04-02T05:14:02.107 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 bash[86424]: 8e478dd4da2dbd4ceb762aa4372bed999d44e3977f4dd14fd300b2ada2762906 2024-04-02T05:14:02.107 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:01 smithi086 systemd[1]: Started Ceph mon.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:14:03.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ignoring --setuser ceph since I am not root 2024-04-02T05:14:03.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ignoring --setgroup ceph since I am not root 2024-04-02T05:14:03.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mgr[62645]: -- 172.21.15.86:0/3789562185 <== mon.0 v2:172.21.15.86:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56487463a5a0 con 0x56487460ec00 2024-04-02T05:14:03.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[86443]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:14:03.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[86443]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:14:03.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[86443]: mon.a calling monitor election 2024-04-02T05:14:03.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[86443]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-02T05:14:03.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[86443]: monmap epoch 3 2024-04-02T05:14:03.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[86443]: fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 2024-04-02T05:14:03.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[86443]: last_changed 2024-04-02T04:45:24.807859+0000 2024-04-02T05:14:03.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[86443]: created 2024-04-02T04:43:48.251359+0000 2024-04-02T05:14:03.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[86443]: min_mon_release 17 (quincy) 2024-04-02T05:14:03.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[86443]: election_strategy: 1 2024-04-02T05:14:03.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[86443]: 0: [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] mon.a 2024-04-02T05:14:03.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[86443]: 1: [v2:172.21.15.86:3301/0,v1:172.21.15.86:6790/0] mon.c 2024-04-02T05:14:03.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[86443]: 2: [v2:172.21.15.192:3300/0,v1:172.21.15.192:6789/0] mon.b 2024-04-02T05:14:03.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[86443]: fsmap 2024-04-02T05:14:03.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[86443]: osdmap e110: 8 total, 8 up, 8 in 2024-04-02T05:14:03.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[86443]: mgrmap e41: y(active, since 3m), standbys: x 2024-04-02T05:14:03.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[86443]: overall HEALTH_OK 2024-04-02T05:14:03.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[86443]: from='mgr.15345 ' entity='' 2024-04-02T05:14:03.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[86443]: mgrmap e42: y(active, since 3m), standbys: x 2024-04-02T05:14:03.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[29505]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:14:03.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[29505]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:14:03.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[29505]: mon.a calling monitor election 2024-04-02T05:14:03.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[29505]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-02T05:14:03.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[29505]: monmap epoch 3 2024-04-02T05:14:03.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[29505]: fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 2024-04-02T05:14:03.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[29505]: last_changed 2024-04-02T04:45:24.807859+0000 2024-04-02T05:14:03.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[29505]: created 2024-04-02T04:43:48.251359+0000 2024-04-02T05:14:03.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[29505]: min_mon_release 17 (quincy) 2024-04-02T05:14:03.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[29505]: election_strategy: 1 2024-04-02T05:14:03.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[29505]: 0: [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] mon.a 2024-04-02T05:14:03.586 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[29505]: 1: [v2:172.21.15.86:3301/0,v1:172.21.15.86:6790/0] mon.c 2024-04-02T05:14:03.586 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[29505]: 2: [v2:172.21.15.192:3300/0,v1:172.21.15.192:6789/0] mon.b 2024-04-02T05:14:03.586 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[29505]: fsmap 2024-04-02T05:14:03.586 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[29505]: osdmap e110: 8 total, 8 up, 8 in 2024-04-02T05:14:03.586 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[29505]: mgrmap e41: y(active, since 3m), standbys: x 2024-04-02T05:14:03.586 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[29505]: overall HEALTH_OK 2024-04-02T05:14:03.586 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[29505]: from='mgr.15345 ' entity='' 2024-04-02T05:14:03.586 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-mon[29505]: mgrmap e42: y(active, since 3m), standbys: x 2024-04-02T05:14:03.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:14:03 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: ignoring --setuser ceph since I am not root 2024-04-02T05:14:03.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:14:03 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: ignoring --setgroup ceph since I am not root 2024-04-02T05:14:03.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:14:03 smithi192 ceph-mgr[61368]: -- 172.21.15.192:0/3938802576 <== mon.2 v2:172.21.15.192:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x556ff7f845a0 con 0x556ff7f58800 2024-04-02T05:14:03.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:03 smithi192 ceph-mon[68000]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:14:03.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:03 smithi192 ceph-mon[68000]: from='mgr.15345 172.21.15.86:0/2136581571' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:14:03.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:03 smithi192 ceph-mon[68000]: mon.a calling monitor election 2024-04-02T05:14:03.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:03 smithi192 ceph-mon[68000]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-02T05:14:03.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:03 smithi192 ceph-mon[68000]: monmap epoch 3 2024-04-02T05:14:03.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:03 smithi192 ceph-mon[68000]: fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 2024-04-02T05:14:03.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:03 smithi192 ceph-mon[68000]: last_changed 2024-04-02T04:45:24.807859+0000 2024-04-02T05:14:03.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:03 smithi192 ceph-mon[68000]: created 2024-04-02T04:43:48.251359+0000 2024-04-02T05:14:03.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:03 smithi192 ceph-mon[68000]: min_mon_release 17 (quincy) 2024-04-02T05:14:03.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:03 smithi192 ceph-mon[68000]: election_strategy: 1 2024-04-02T05:14:03.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:03 smithi192 ceph-mon[68000]: 0: [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] mon.a 2024-04-02T05:14:03.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:03 smithi192 ceph-mon[68000]: 1: [v2:172.21.15.86:3301/0,v1:172.21.15.86:6790/0] mon.c 2024-04-02T05:14:03.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:03 smithi192 ceph-mon[68000]: 2: [v2:172.21.15.192:3300/0,v1:172.21.15.192:6789/0] mon.b 2024-04-02T05:14:03.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:03 smithi192 ceph-mon[68000]: fsmap 2024-04-02T05:14:03.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:03 smithi192 ceph-mon[68000]: osdmap e110: 8 total, 8 up, 8 in 2024-04-02T05:14:03.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:03 smithi192 ceph-mon[68000]: mgrmap e41: y(active, since 3m), standbys: x 2024-04-02T05:14:03.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:03 smithi192 ceph-mon[68000]: overall HEALTH_OK 2024-04-02T05:14:03.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:03 smithi192 ceph-mon[68000]: from='mgr.15345 ' entity='' 2024-04-02T05:14:03.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:03 smithi192 ceph-mon[68000]: mgrmap e42: y(active, since 3m), standbys: x 2024-04-02T05:14:03.935 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:14:03 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:14:03.838+0000 7f3ac81d5200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-02T05:14:04.333 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:14:03.842+0000 7fe1e6e68200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-02T05:14:04.333 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:14:03.935+0000 7fe1e6e68200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-02T05:14:04.427 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:14:03 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:14:03.933+0000 7f3ac81d5200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-02T05:14:04.832 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:14:04.574+0000 7fe1e6e68200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-02T05:14:04.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:14:04.668+0000 7fe1e6e68200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-02T05:14:04.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:14:04.813+0000 7fe1e6e68200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-02T05:14:04.912 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:14:04 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:14:04.576+0000 7f3ac81d5200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-02T05:14:04.913 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:14:04 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:14:04.668+0000 7f3ac81d5200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-02T05:14:04.913 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:14:04 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:14:04.813+0000 7f3ac81d5200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-02T05:14:05.177 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:14:04 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:14:04.909+0000 7f3ac81d5200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-02T05:14:05.177 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:14:04 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:14:04.993+0000 7f3ac81d5200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-02T05:14:05.267 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:14:04.914+0000 7fe1e6e68200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-02T05:14:05.267 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:05 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:14:04.999+0000 7fe1e6e68200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-02T05:14:05.267 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:05 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:14:05.263+0000 7fe1e6e68200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-02T05:14:05.555 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:05 smithi192 ceph-mon[68000]: mgrmap e43: y(active, since 3m), standbys: x 2024-04-02T05:14:05.555 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:14:05 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:14:05.258+0000 7f3ac81d5200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-02T05:14:05.555 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:14:05 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:14:05.369+0000 7f3ac81d5200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-02T05:14:05.555 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:14:05 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:14:05.457+0000 7f3ac81d5200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-02T05:14:05.555 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:14:05 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:14:05.552+0000 7f3ac81d5200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-02T05:14:05.581 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:05 smithi086 ceph-mon[29505]: mgrmap e43: y(active, since 3m), standbys: x 2024-04-02T05:14:05.582 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:05 smithi086 ceph-mon[86443]: mgrmap e43: y(active, since 3m), standbys: x 2024-04-02T05:14:05.582 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:05 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:14:05.376+0000 7fe1e6e68200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-02T05:14:05.582 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:05 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:14:05.479+0000 7fe1e6e68200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-02T05:14:05.832 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:05 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:14:05.576+0000 7fe1e6e68200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-02T05:14:05.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:05 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:14:05.755+0000 7fe1e6e68200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-02T05:14:05.927 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:14:05 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:14:05.732+0000 7f3ac81d5200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-02T05:14:07.083 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:06 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:14:06.718+0000 7fe1e6e68200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-02T05:14:07.083 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:06 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:14:06.857+0000 7fe1e6e68200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-02T05:14:07.177 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:14:06 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:14:06.694+0000 7f3ac81d5200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-02T05:14:07.177 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:14:06 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:14:06.827+0000 7f3ac81d5200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-02T05:14:07.483 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:14:07 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:14:07.187+0000 7f3ac81d5200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-02T05:14:07.483 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:14:07 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:14:07.280+0000 7f3ac81d5200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-02T05:14:07.483 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:14:07 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:14:07.480+0000 7f3ac81d5200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-02T05:14:07.495 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:07 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:14:07.198+0000 7fe1e6e68200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-02T05:14:07.495 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:07 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:14:07.290+0000 7fe1e6e68200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-02T05:14:07.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:07 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:14:07.491+0000 7fe1e6e68200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-02T05:14:07.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:07 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:14:07.681+0000 7fe1e6e68200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-02T05:14:07.927 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:14:07 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:14:07.666+0000 7f3ac81d5200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-02T05:14:08.224 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:14:07 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:14:07.957+0000 7f3ac81d5200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-02T05:14:08.244 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:07 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:14:07.975+0000 7fe1e6e68200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-02T05:14:08.582 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:14:08.241+0000 7fe1e6e68200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-02T05:14:08.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:14:08.484+0000 7fe1e6e68200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-02T05:14:08.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:14:08 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:14:08.221+0000 7f3ac81d5200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-02T05:14:08.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:14:08 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:14:08.463+0000 7f3ac81d5200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-02T05:14:09.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:09 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:14:09.308+0000 7fe1e6e68200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-02T05:14:09.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:14:09 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:14:09.288+0000 7f3ac81d5200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-02T05:14:10.177 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:14:09 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:14:09.865+0000 7f3ac81d5200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-02T05:14:10.478 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:14:09 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:14:09.965+0000 7f3ac81d5200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-02T05:14:10.479 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:09 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:14:09.886+0000 7fe1e6e68200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-02T05:14:10.480 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:09 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:14:09.989+0000 7fe1e6e68200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-02T05:14:10.480 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:14:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:14:10.191+0000 7f3ac81d5200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-02T05:14:10.480 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:14:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: [02/Apr/2024:05:14:10] ENGINE Bus STARTING 2024-04-02T05:14:10.480 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:14:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: CherryPy Checker: 2024-04-02T05:14:10.480 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:14:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: The Application mounted at '' has an empty config. 2024-04-02T05:14:10.480 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:14:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: 2024-04-02T05:14:10.481 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:14:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: [02/Apr/2024:05:14:10] ENGINE Serving on http://:::9283 2024-04-02T05:14:10.481 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:14:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x[61364]: [02/Apr/2024:05:14:10] ENGINE Bus STARTED 2024-04-02T05:14:10.481 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:10 smithi192 ceph-mon[68000]: Standby manager daemon x restarted 2024-04-02T05:14:10.481 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:10 smithi192 ceph-mon[68000]: Standby manager daemon x started 2024-04-02T05:14:10.481 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:10 smithi192 ceph-mon[68000]: from='mgr.? 172.21.15.192:0/3079154369' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-02T05:14:10.482 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:10 smithi192 ceph-mon[68000]: from='mgr.? 172.21.15.192:0/3079154369' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T05:14:10.482 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:10 smithi192 ceph-mon[68000]: from='mgr.? 172.21.15.192:0/3079154369' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-02T05:14:10.482 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:10 smithi192 ceph-mon[68000]: from='mgr.? 172.21.15.192:0/3079154369' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T05:14:10.482 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:10 smithi192 ceph-mon[68000]: Active manager daemon y restarted 2024-04-02T05:14:10.482 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:10 smithi192 ceph-mon[68000]: Activating manager daemon y 2024-04-02T05:14:10.482 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:10 smithi192 ceph-mon[68000]: osdmap e111: 8 total, 8 up, 8 in 2024-04-02T05:14:10.483 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:10 smithi192 ceph-mon[68000]: mgrmap e44: y(active, starting, since 0.00762317s), standbys: x 2024-04-02T05:14:10.483 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:10 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T05:14:10.483 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:10 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T05:14:10.483 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:10 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T05:14:10.483 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:10 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T05:14:10.484 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:10 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T05:14:10.484 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:10 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T05:14:10.484 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:10 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T05:14:10.484 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:10 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T05:14:10.484 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:10 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T05:14:10.484 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:10 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T05:14:10.484 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:10 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T05:14:10.485 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:10 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T05:14:10.485 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:10 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T05:14:10.485 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:10 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T05:14:10.485 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:10 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T05:14:10.485 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:10 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T05:14:10.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:14:10.216+0000 7fe1e6e68200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-02T05:14:10.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: [02/Apr/2024:05:14:10] ENGINE Bus STARTING 2024-04-02T05:14:10.584 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: CherryPy Checker: 2024-04-02T05:14:10.584 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: The Application mounted at '' has an empty config. 2024-04-02T05:14:10.584 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:14:10.584 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: [02/Apr/2024:05:14:10] ENGINE Serving on http://:::9283 2024-04-02T05:14:10.584 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: [02/Apr/2024:05:14:10] ENGINE Bus STARTED 2024-04-02T05:14:10.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[86443]: Standby manager daemon x restarted 2024-04-02T05:14:10.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[86443]: Standby manager daemon x started 2024-04-02T05:14:10.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[86443]: from='mgr.? 172.21.15.192:0/3079154369' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-02T05:14:10.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[86443]: from='mgr.? 172.21.15.192:0/3079154369' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T05:14:10.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[86443]: from='mgr.? 172.21.15.192:0/3079154369' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-02T05:14:10.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[86443]: from='mgr.? 172.21.15.192:0/3079154369' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T05:14:10.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[86443]: Active manager daemon y restarted 2024-04-02T05:14:10.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[86443]: Activating manager daemon y 2024-04-02T05:14:10.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[86443]: osdmap e111: 8 total, 8 up, 8 in 2024-04-02T05:14:10.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[86443]: mgrmap e44: y(active, starting, since 0.00762317s), standbys: x 2024-04-02T05:14:10.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T05:14:10.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T05:14:10.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T05:14:10.587 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T05:14:10.587 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T05:14:10.587 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T05:14:10.587 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T05:14:10.587 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T05:14:10.587 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T05:14:10.587 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T05:14:10.587 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T05:14:10.588 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T05:14:10.588 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T05:14:10.588 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T05:14:10.588 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T05:14:10.588 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T05:14:10.589 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-c[29501]: 2024-04-02T05:14:10.223+0000 7fa35a65c700 -1 mon.c@1(peon).osd e111 update_from_paxos full map CRC mismatch, resetting to canonical 2024-04-02T05:14:10.589 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[29505]: Standby manager daemon x restarted 2024-04-02T05:14:10.589 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[29505]: Standby manager daemon x started 2024-04-02T05:14:10.589 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.192:0/3079154369' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-02T05:14:10.589 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.192:0/3079154369' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T05:14:10.589 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.192:0/3079154369' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-02T05:14:10.589 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[29505]: from='mgr.? 172.21.15.192:0/3079154369' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T05:14:10.590 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[29505]: Active manager daemon y restarted 2024-04-02T05:14:10.590 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[29505]: Activating manager daemon y 2024-04-02T05:14:10.590 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[29505]: osdmap e111: 8 total, 8 up, 8 in 2024-04-02T05:14:10.590 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[29505]: mgrmap e44: y(active, starting, since 0.00762317s), standbys: x 2024-04-02T05:14:10.590 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[29505]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T05:14:10.591 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[29505]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T05:14:10.591 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[29505]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T05:14:10.591 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[29505]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T05:14:10.591 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[29505]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T05:14:10.591 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[29505]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T05:14:10.591 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[29505]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T05:14:10.591 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[29505]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T05:14:10.591 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[29505]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T05:14:10.592 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[29505]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T05:14:10.592 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[29505]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T05:14:10.592 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[29505]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T05:14:10.592 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[29505]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T05:14:10.592 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[29505]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T05:14:10.592 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[29505]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T05:14:10.592 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:10 smithi086 ceph-mon[29505]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T05:14:11.556 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:11 smithi192 ceph-mon[68000]: failed to encode map e111 with expected crc 2024-04-02T05:14:11.556 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:11 smithi192 ceph-mon[68000]: failed to encode map e111 with expected crc 2024-04-02T05:14:11.556 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:11 smithi192 ceph-mon[68000]: Manager daemon y is now available 2024-04-02T05:14:11.556 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:11 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:14:11.556 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:11 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T05:14:11.556 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:11 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T05:14:11.556 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:11 smithi192 ceph-mon[68000]: failed to encode map e111 with expected crc 2024-04-02T05:14:11.556 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:11 smithi192 ceph-mon[68000]: failed to encode map e111 with expected crc 2024-04-02T05:14:11.556 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:11 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T05:14:11.556 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:11 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T05:14:11.557 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:11 smithi192 ceph-mon[68000]: failed to encode map e111 with expected crc 2024-04-02T05:14:11.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:14:11.258+0000 7fe19d012700 -1 mgr.server handle_report got status from non-daemon mon.a 2024-04-02T05:14:11.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:11 smithi086 ceph-mon[29505]: failed to encode map e111 with expected crc 2024-04-02T05:14:11.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:11 smithi086 ceph-mon[29505]: failed to encode map e111 with expected crc 2024-04-02T05:14:11.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:11 smithi086 ceph-mon[29505]: Manager daemon y is now available 2024-04-02T05:14:11.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:11 smithi086 ceph-mon[29505]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:14:11.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:11 smithi086 ceph-mon[29505]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T05:14:11.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:11 smithi086 ceph-mon[29505]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T05:14:11.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:11 smithi086 ceph-mon[29505]: failed to encode map e111 with expected crc 2024-04-02T05:14:11.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:11 smithi086 ceph-mon[29505]: failed to encode map e111 with expected crc 2024-04-02T05:14:11.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:11 smithi086 ceph-mon[29505]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T05:14:11.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:11 smithi086 ceph-mon[29505]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T05:14:11.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:11 smithi086 ceph-mon[29505]: failed to encode map e111 with expected crc 2024-04-02T05:14:11.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:11 smithi086 ceph-mon[86443]: failed to encode map e111 with expected crc 2024-04-02T05:14:11.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:11 smithi086 ceph-mon[86443]: failed to encode map e111 with expected crc 2024-04-02T05:14:11.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:11 smithi086 ceph-mon[86443]: Manager daemon y is now available 2024-04-02T05:14:11.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:11 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:14:11.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:11 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T05:14:11.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:11 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T05:14:11.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:11 smithi086 ceph-mon[86443]: failed to encode map e111 with expected crc 2024-04-02T05:14:11.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:11 smithi086 ceph-mon[86443]: failed to encode map e111 with expected crc 2024-04-02T05:14:11.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:11 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T05:14:11.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:11 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T05:14:11.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:11 smithi086 ceph-mon[86443]: failed to encode map e111 with expected crc 2024-04-02T05:14:12.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:12 smithi086 ceph-mon[29505]: failed to encode map e111 with expected crc 2024-04-02T05:14:12.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:12 smithi086 ceph-mon[29505]: [02/Apr/2024:05:14:11] ENGINE Bus STARTING 2024-04-02T05:14:12.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:12 smithi086 ceph-mon[29505]: mgrmap e45: y(active, since 1.03731s), standbys: x 2024-04-02T05:14:12.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:12 smithi086 ceph-mon[29505]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:14:12.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:12 smithi086 ceph-mon[29505]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:14:12.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:12 smithi086 ceph-mon[29505]: [02/Apr/2024:05:14:11] ENGINE Serving on http://172.21.15.86:8765 2024-04-02T05:14:12.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:12 smithi086 ceph-mon[86443]: failed to encode map e111 with expected crc 2024-04-02T05:14:12.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:12 smithi086 ceph-mon[86443]: [02/Apr/2024:05:14:11] ENGINE Bus STARTING 2024-04-02T05:14:12.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:12 smithi086 ceph-mon[86443]: mgrmap e45: y(active, since 1.03731s), standbys: x 2024-04-02T05:14:12.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:12 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:14:12.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:12 smithi086 ceph-mon[86443]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:14:12.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:12 smithi086 ceph-mon[86443]: [02/Apr/2024:05:14:11] ENGINE Serving on http://172.21.15.86:8765 2024-04-02T05:14:12.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:12 smithi192 ceph-mon[68000]: failed to encode map e111 with expected crc 2024-04-02T05:14:12.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:12 smithi192 ceph-mon[68000]: [02/Apr/2024:05:14:11] ENGINE Bus STARTING 2024-04-02T05:14:12.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:12 smithi192 ceph-mon[68000]: mgrmap e45: y(active, since 1.03731s), standbys: x 2024-04-02T05:14:12.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:12 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:14:12.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:12 smithi192 ceph-mon[68000]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:14:12.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:12 smithi192 ceph-mon[68000]: [02/Apr/2024:05:14:11] ENGINE Serving on http://172.21.15.86:8765 2024-04-02T05:14:13.404 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:13 smithi086 ceph-mon[29505]: [02/Apr/2024:05:14:11] ENGINE Serving on https://172.21.15.86:7150 2024-04-02T05:14:13.404 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:13 smithi086 ceph-mon[29505]: [02/Apr/2024:05:14:11] ENGINE Bus STARTED 2024-04-02T05:14:13.404 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:13 smithi086 ceph-mon[29505]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:14:13.404 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:13 smithi086 ceph-mon[29505]: mgrmap e46: y(active, since 2s), standbys: x 2024-04-02T05:14:13.405 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:13 smithi086 ceph-mon[86443]: [02/Apr/2024:05:14:11] ENGINE Serving on https://172.21.15.86:7150 2024-04-02T05:14:13.405 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:13 smithi086 ceph-mon[86443]: [02/Apr/2024:05:14:11] ENGINE Bus STARTED 2024-04-02T05:14:13.405 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:13 smithi086 ceph-mon[86443]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:14:13.405 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:13 smithi086 ceph-mon[86443]: mgrmap e46: y(active, since 2s), standbys: x 2024-04-02T05:14:13.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:13 smithi192 ceph-mon[68000]: [02/Apr/2024:05:14:11] ENGINE Serving on https://172.21.15.86:7150 2024-04-02T05:14:13.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:13 smithi192 ceph-mon[68000]: [02/Apr/2024:05:14:11] ENGINE Bus STARTED 2024-04-02T05:14:13.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:13 smithi192 ceph-mon[68000]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:14:13.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:13 smithi192 ceph-mon[68000]: mgrmap e46: y(active, since 2s), standbys: x 2024-04-02T05:14:13.783 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:13 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:14:13] "GET /metrics HTTP/1.1" 200 35208 "" "Prometheus/2.43.0" 2024-04-02T05:14:15.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:15 smithi086 ceph-mon[86443]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:14:15.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:15 smithi086 ceph-mon[86443]: mgrmap e47: y(active, since 4s), standbys: x 2024-04-02T05:14:15.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:15 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:15.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:15 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:15.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:15 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:15.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:15 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:15.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:15 smithi086 ceph-mon[29505]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:14:15.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:15 smithi086 ceph-mon[29505]: mgrmap e47: y(active, since 4s), standbys: x 2024-04-02T05:14:15.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:15 smithi086 ceph-mon[29505]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:15.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:15 smithi086 ceph-mon[29505]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:15.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:15 smithi086 ceph-mon[29505]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:15.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:15 smithi086 ceph-mon[29505]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:15.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:15 smithi192 ceph-mon[68000]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:14:15.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:15 smithi192 ceph-mon[68000]: mgrmap e47: y(active, since 4s), standbys: x 2024-04-02T05:14:15.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:15 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:15.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:15 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:15.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:15 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:15.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:15 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:17.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:16 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:17.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:16 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:17.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:16 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:17.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:16 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:17.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:16 smithi192 ceph-mon[68000]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:14:17.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:16 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:17.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:16 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:17.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:16 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:17.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:16 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:17.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:16 smithi086 ceph-mon[86443]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:14:17.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:16 smithi086 ceph-mon[29505]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:17.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:16 smithi086 ceph-mon[29505]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:17.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:16 smithi086 ceph-mon[29505]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:17.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:16 smithi086 ceph-mon[29505]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:17.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:16 smithi086 ceph-mon[29505]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:14:19.349 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:19 smithi192 ceph-mon[68000]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-02T05:14:19.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:19 smithi086 ceph-mon[29505]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-02T05:14:19.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:19 smithi086 ceph-mon[86443]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-02T05:14:20.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:20 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:20.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:20 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:20.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:20 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:20.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:20 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:14:20.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:20 smithi192 ceph-mon[68000]: Updating smithi086:/etc/ceph/ceph.conf 2024-04-02T05:14:20.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:20 smithi192 ceph-mon[68000]: Updating smithi192:/etc/ceph/ceph.conf 2024-04-02T05:14:20.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:20 smithi192 ceph-mon[68000]: Updating smithi192:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T05:14:20.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:20 smithi192 ceph-mon[68000]: Updating smithi086:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T05:14:20.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:20 smithi192 ceph-mon[68000]: Updating smithi192:/etc/ceph/ceph.client.admin.keyring 2024-04-02T05:14:20.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:20 smithi192 ceph-mon[68000]: Updating smithi086:/etc/ceph/ceph.client.admin.keyring 2024-04-02T05:14:20.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:20 smithi192 ceph-mon[68000]: Updating smithi192:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-02T05:14:20.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:20 smithi192 ceph-mon[68000]: Updating smithi086:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-02T05:14:20.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:20 smithi192 ceph-mon[68000]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-02T05:14:20.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:20 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:20.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:20 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:20.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:20 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:20.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:20 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:20.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:20 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:20.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:20 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:20.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:20 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:20.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:20 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:20.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:20 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:14:20.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:20 smithi086 ceph-mon[86443]: Updating smithi086:/etc/ceph/ceph.conf 2024-04-02T05:14:20.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:20 smithi086 ceph-mon[86443]: Updating smithi192:/etc/ceph/ceph.conf 2024-04-02T05:14:20.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:20 smithi086 ceph-mon[86443]: Updating smithi192:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T05:14:20.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:20 smithi086 ceph-mon[86443]: Updating smithi086:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T05:14:20.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:20 smithi086 ceph-mon[86443]: Updating smithi192:/etc/ceph/ceph.client.admin.keyring 2024-04-02T05:14:20.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:20 smithi086 ceph-mon[86443]: Updating smithi086:/etc/ceph/ceph.client.admin.keyring 2024-04-02T05:14:20.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:20 smithi086 ceph-mon[86443]: Updating smithi192:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-02T05:14:20.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:20 smithi086 ceph-mon[86443]: Updating smithi086:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-02T05:14:20.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:20 smithi086 ceph-mon[86443]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-02T05:14:20.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:20 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:20.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:20 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:20.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:20 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:20.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:20 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:20.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:20 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:20.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:20 smithi086 ceph-mon[29505]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:20.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:20 smithi086 ceph-mon[29505]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:20.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:20 smithi086 ceph-mon[29505]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:20.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:20 smithi086 ceph-mon[29505]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:14:20.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:20 smithi086 ceph-mon[29505]: Updating smithi086:/etc/ceph/ceph.conf 2024-04-02T05:14:20.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:20 smithi086 ceph-mon[29505]: Updating smithi192:/etc/ceph/ceph.conf 2024-04-02T05:14:20.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:20 smithi086 ceph-mon[29505]: Updating smithi192:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T05:14:20.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:20 smithi086 ceph-mon[29505]: Updating smithi086:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T05:14:20.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:20 smithi086 ceph-mon[29505]: Updating smithi192:/etc/ceph/ceph.client.admin.keyring 2024-04-02T05:14:20.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:20 smithi086 ceph-mon[29505]: Updating smithi086:/etc/ceph/ceph.client.admin.keyring 2024-04-02T05:14:20.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:20 smithi086 ceph-mon[29505]: Updating smithi192:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-02T05:14:20.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:20 smithi086 ceph-mon[29505]: Updating smithi086:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-02T05:14:20.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:20 smithi086 ceph-mon[29505]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-02T05:14:20.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:20 smithi086 ceph-mon[29505]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:20.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:20 smithi086 ceph-mon[29505]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:20.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:20 smithi086 ceph-mon[29505]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:20.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:20 smithi086 ceph-mon[29505]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:20.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:20 smithi086 ceph-mon[29505]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:21.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:21 smithi086 ceph-mon[29505]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:14:21.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:21 smithi086 ceph-mon[29505]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:14:21.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:21 smithi086 ceph-mon[29505]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-02T05:14:21.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:21 smithi086 ceph-mon[29505]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-02T05:14:21.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:21 smithi086 ceph-mon[29505]: Upgrade: It appears safe to stop mon.c 2024-04-02T05:14:21.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:21 smithi086 ceph-mon[29505]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:14:21.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:21 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:14:21.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:21 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:14:21.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:21 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-02T05:14:21.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:21 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-02T05:14:21.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:21 smithi086 ceph-mon[86443]: Upgrade: It appears safe to stop mon.c 2024-04-02T05:14:21.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:21 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:14:21.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:21 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:14:21.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:21 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:14:21.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:21 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-02T05:14:21.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:21 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-02T05:14:21.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:21 smithi192 ceph-mon[68000]: Upgrade: It appears safe to stop mon.c 2024-04-02T05:14:21.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:21 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:14:23.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:22 smithi086 ceph-mon[86443]: Upgrade: Updating mon.c 2024-04-02T05:14:23.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:22 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:23.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:22 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T05:14:23.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:22 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T05:14:23.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:22 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:23.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:22 smithi086 ceph-mon[86443]: Deploying daemon mon.c on smithi086 2024-04-02T05:14:23.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:22 smithi086 ceph-mon[86443]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-02T05:14:23.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:22 smithi086 ceph-mon[29505]: Upgrade: Updating mon.c 2024-04-02T05:14:23.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:22 smithi086 ceph-mon[29505]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:23.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:22 smithi086 ceph-mon[29505]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T05:14:23.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:22 smithi086 ceph-mon[29505]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T05:14:23.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:22 smithi086 ceph-mon[29505]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:23.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:22 smithi086 ceph-mon[29505]: Deploying daemon mon.c on smithi086 2024-04-02T05:14:23.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:22 smithi086 ceph-mon[29505]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-02T05:14:23.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:22 smithi192 ceph-mon[68000]: Upgrade: Updating mon.c 2024-04-02T05:14:23.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:22 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:23.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:22 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T05:14:23.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:22 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T05:14:23.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:22 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:23.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:22 smithi192 ceph-mon[68000]: Deploying daemon mon.c on smithi086 2024-04-02T05:14:23.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:22 smithi192 ceph-mon[68000]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-02T05:14:23.668 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:14:23] "GET /metrics HTTP/1.1" 200 35208 "" "Prometheus/2.43.0" 2024-04-02T05:14:23.927 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:23 smithi086 systemd[1]: Stopping Ceph mon.c for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:14:23.927 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-c[29501]: 2024-04-02T05:14:23.912+0000 7fa35fe67700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T05:14:23.927 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-c[29501]: 2024-04-02T05:14:23.912+0000 7fa35fe67700 -1 mon.c@1(peon) e3 *** Got Signal Terminated *** 2024-04-02T05:14:24.231 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:24 smithi086 podman[88653]: 2024-04-02 05:14:24.05660698 +0000 UTC m=+0.251685969 container died 49275483a2d4cac9eac2588fcaecffbe3e68069c9b9a80ecebd7d9d6b8c49808 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-c, io.buildah.version=1.19.8, release=754, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, GIT_CLEAN=True, name=centos-stream, vcs-type=git, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True) 2024-04-02T05:14:24.232 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: [02/Apr/2024:05:14:23] ENGINE Bus STOPPING 2024-04-02T05:14:24.232 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: [02/Apr/2024:05:14:23] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-02T05:14:24.232 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: [02/Apr/2024:05:14:23] ENGINE Bus STOPPED 2024-04-02T05:14:24.232 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: [02/Apr/2024:05:14:23] ENGINE Bus STARTING 2024-04-02T05:14:24.232 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:24 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: [02/Apr/2024:05:14:24] ENGINE Serving on http://:::9283 2024-04-02T05:14:24.232 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:24 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: [02/Apr/2024:05:14:24] ENGINE Bus STARTED 2024-04-02T05:14:24.565 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:24 smithi086 podman[88653]: 2024-04-02 05:14:24.37890864 +0000 UTC m=+0.573987629 container cleanup 49275483a2d4cac9eac2588fcaecffbe3e68069c9b9a80ecebd7d9d6b8c49808 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-c, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, io.openshift.expose-services=, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., distribution-scope=public, release=754, GIT_CLEAN=True) 2024-04-02T05:14:24.565 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:24 smithi086 bash[88653]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-c 2024-04-02T05:14:24.819 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:24 smithi086 podman[88678]: 2024-04-02 05:14:24.563646654 +0000 UTC m=+0.501852108 container remove 49275483a2d4cac9eac2588fcaecffbe3e68069c9b9a80ecebd7d9d6b8c49808 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-c, architecture=x86_64, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0) 2024-04-02T05:14:25.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:24 smithi086 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mon.c.service: Deactivated successfully. 2024-04-02T05:14:25.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:24 smithi086 systemd[1]: Stopped Ceph mon.c for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:14:25.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:24 smithi086 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mon.c.service: Consumed 14.908s CPU time. 2024-04-02T05:14:25.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 systemd[1]: Starting Ceph mon.c for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:14:25.510 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 podman[88819]: 2024-04-02 05:14:25.152215289 +0000 UTC m=+0.018103076 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:14:25.511 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 podman[88819]: 2024-04-02 05:14:25.264388675 +0000 UTC m=+0.130276464 container create 8180999b3142bc0604f44ab3bc63470afcb15ad9f18f162b326a462deb1d619d (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-c, GIT_CLEAN=True, RELEASE=squid-6cec793, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-04-02T05:14:25.511 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 podman[88819]: 2024-04-02 05:14:25.508749571 +0000 UTC m=+0.374637351 container init 8180999b3142bc0604f44ab3bc63470afcb15ad9f18f162b326a462deb1d619d (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-c, RELEASE=squid-6cec793, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True) 2024-04-02T05:14:25.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 podman[88819]: 2024-04-02 05:14:25.512472883 +0000 UTC m=+0.378360672 container start 8180999b3142bc0604f44ab3bc63470afcb15ad9f18f162b326a462deb1d619d (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-c, org.label-schema.schema-version=1.0, ceph=True, RELEASE=squid-6cec793, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-04-02T05:14:25.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: set uid:gid to 167:167 (ceph:ceph) 2024-04-02T05:14:25.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev), process ceph-mon, pid 2 2024-04-02T05:14:25.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: pidfile_write: ignore empty --pid-file 2024-04-02T05:14:25.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: load: jerasure load: lrc 2024-04-02T05:14:25.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: RocksDB version: 7.9.2 2024-04-02T05:14:25.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Git sha 0 2024-04-02T05:14:25.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Compile date 2024-04-01 15:23:26 2024-04-02T05:14:25.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: DB SUMMARY 2024-04-02T05:14:25.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: DB Session ID: 4UN4BSUM4MBR3CS6IERM 2024-04-02T05:14:25.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: CURRENT file: CURRENT 2024-04-02T05:14:25.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: IDENTITY file: IDENTITY 2024-04-02T05:14:25.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: MANIFEST file: MANIFEST-000009 size: 3196 Bytes 2024-04-02T05:14:25.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000060.sst 2024-04-02T05:14:25.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000058.log size: 2277483 ; 2024-04-02T05:14:25.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.error_if_exists: 0 2024-04-02T05:14:25.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.create_if_missing: 0 2024-04-02T05:14:25.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.paranoid_checks: 1 2024-04-02T05:14:25.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-02T05:14:25.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-02T05:14:25.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-02T05:14:25.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.env: 0x561cac0dec60 2024-04-02T05:14:25.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.fs: PosixFileSystem 2024-04-02T05:14:25.837 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.info_log: 0x561cadfffae0 2024-04-02T05:14:25.837 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.max_file_opening_threads: 16 2024-04-02T05:14:25.837 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.statistics: (nil) 2024-04-02T05:14:25.837 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.use_fsync: 0 2024-04-02T05:14:25.837 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.max_log_file_size: 0 2024-04-02T05:14:25.837 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-02T05:14:25.837 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-02T05:14:25.838 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.keep_log_file_num: 1000 2024-04-02T05:14:25.838 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.recycle_log_file_num: 0 2024-04-02T05:14:25.840 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.allow_fallocate: 1 2024-04-02T05:14:25.841 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.allow_mmap_reads: 0 2024-04-02T05:14:25.845 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.allow_mmap_writes: 0 2024-04-02T05:14:25.845 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.use_direct_reads: 0 2024-04-02T05:14:25.845 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-02T05:14:25.845 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.create_missing_column_families: 0 2024-04-02T05:14:25.845 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.db_log_dir: 2024-04-02T05:14:25.845 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.wal_dir: 2024-04-02T05:14:25.845 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-02T05:14:25.845 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-02T05:14:25.845 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-02T05:14:25.845 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-02T05:14:25.845 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-02T05:14:25.846 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-02T05:14:25.846 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.advise_random_on_open: 1 2024-04-02T05:14:25.846 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.db_write_buffer_size: 0 2024-04-02T05:14:25.846 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.write_buffer_manager: 0x561cadfd2820 2024-04-02T05:14:25.846 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-02T05:14:25.847 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-02T05:14:25.847 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-02T05:14:25.847 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.rate_limiter: (nil) 2024-04-02T05:14:25.847 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-02T05:14:25.847 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.wal_recovery_mode: 2 2024-04-02T05:14:25.847 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.enable_thread_tracking: 0 2024-04-02T05:14:25.847 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.enable_pipelined_write: 0 2024-04-02T05:14:25.847 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.unordered_write: 0 2024-04-02T05:14:25.847 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-02T05:14:25.847 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-02T05:14:25.847 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-02T05:14:25.847 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-02T05:14:25.847 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.row_cache: None 2024-04-02T05:14:25.847 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.wal_filter: None 2024-04-02T05:14:25.847 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-02T05:14:25.848 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.allow_ingest_behind: 0 2024-04-02T05:14:25.848 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.two_write_queues: 0 2024-04-02T05:14:25.848 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.manual_wal_flush: 0 2024-04-02T05:14:25.849 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.wal_compression: 0 2024-04-02T05:14:25.849 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.atomic_flush: 0 2024-04-02T05:14:25.849 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-02T05:14:25.849 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-02T05:14:25.849 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-02T05:14:25.849 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.log_readahead_size: 0 2024-04-02T05:14:25.849 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-02T05:14:25.849 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.best_efforts_recovery: 0 2024-04-02T05:14:25.849 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-02T05:14:25.849 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-02T05:14:25.849 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.allow_data_in_errors: 0 2024-04-02T05:14:25.849 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.db_host_id: __hostname__ 2024-04-02T05:14:25.849 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-02T05:14:25.849 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.max_background_jobs: 2 2024-04-02T05:14:25.849 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.max_background_compactions: -1 2024-04-02T05:14:25.850 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.max_subcompactions: 1 2024-04-02T05:14:25.850 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-02T05:14:25.850 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-02T05:14:25.850 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-02T05:14:25.850 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.max_total_wal_size: 0 2024-04-02T05:14:25.850 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-02T05:14:25.850 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-02T05:14:25.850 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-02T05:14:25.850 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-02T05:14:25.850 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.max_open_files: -1 2024-04-02T05:14:25.850 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.bytes_per_sync: 0 2024-04-02T05:14:25.850 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-02T05:14:25.850 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-02T05:14:25.850 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.compaction_readahead_size: 0 2024-04-02T05:14:25.850 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.max_background_flushes: -1 2024-04-02T05:14:25.851 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Compression algorithms supported: 2024-04-02T05:14:25.851 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-02T05:14:25.851 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: kZSTD supported: 0 2024-04-02T05:14:25.851 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: kXpressCompression supported: 0 2024-04-02T05:14:25.851 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: kLZ4HCCompression supported: 1 2024-04-02T05:14:25.851 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: kZlibCompression supported: 1 2024-04-02T05:14:25.851 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: kSnappyCompression supported: 1 2024-04-02T05:14:25.851 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: kLZ4Compression supported: 1 2024-04-02T05:14:25.851 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: kBZip2Compression supported: 0 2024-04-02T05:14:25.851 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-02T05:14:25.851 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-02T05:14:25.851 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-04-02T05:14:25.851 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-02T05:14:25.851 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-02T05:14:25.851 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.merge_operator: 2024-04-02T05:14:25.852 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.compaction_filter: None 2024-04-02T05:14:25.852 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.compaction_filter_factory: None 2024-04-02T05:14:25.852 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.sst_partitioner_factory: None 2024-04-02T05:14:25.852 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-02T05:14:25.852 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-02T05:14:25.852 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x561cadfffbe0) 2024-04-02T05:14:25.852 INFO:journalctl@ceph.mon.c.smithi086.stdout: cache_index_and_filter_blocks: 1 2024-04-02T05:14:25.852 INFO:journalctl@ceph.mon.c.smithi086.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-02T05:14:25.852 INFO:journalctl@ceph.mon.c.smithi086.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-02T05:14:25.853 INFO:journalctl@ceph.mon.c.smithi086.stdout: pin_top_level_index_and_filter: 1 2024-04-02T05:14:25.853 INFO:journalctl@ceph.mon.c.smithi086.stdout: index_type: 0 2024-04-02T05:14:25.853 INFO:journalctl@ceph.mon.c.smithi086.stdout: data_block_index_type: 0 2024-04-02T05:14:25.853 INFO:journalctl@ceph.mon.c.smithi086.stdout: index_shortening: 1 2024-04-02T05:14:25.853 INFO:journalctl@ceph.mon.c.smithi086.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-02T05:14:25.853 INFO:journalctl@ceph.mon.c.smithi086.stdout: checksum: 4 2024-04-02T05:14:25.853 INFO:journalctl@ceph.mon.c.smithi086.stdout: no_block_cache: 0 2024-04-02T05:14:25.853 INFO:journalctl@ceph.mon.c.smithi086.stdout: block_cache: 0x561cae003090 2024-04-02T05:14:25.853 INFO:journalctl@ceph.mon.c.smithi086.stdout: block_cache_name: BinnedLRUCache 2024-04-02T05:14:25.853 INFO:journalctl@ceph.mon.c.smithi086.stdout: block_cache_options: 2024-04-02T05:14:25.853 INFO:journalctl@ceph.mon.c.smithi086.stdout: capacity : 536870912 2024-04-02T05:14:25.853 INFO:journalctl@ceph.mon.c.smithi086.stdout: num_shard_bits : 4 2024-04-02T05:14:25.853 INFO:journalctl@ceph.mon.c.smithi086.stdout: strict_capacity_limit : 0 2024-04-02T05:14:25.853 INFO:journalctl@ceph.mon.c.smithi086.stdout: high_pri_pool_ratio: 0.000 2024-04-02T05:14:25.854 INFO:journalctl@ceph.mon.c.smithi086.stdout: block_cache_compressed: (nil) 2024-04-02T05:14:25.854 INFO:journalctl@ceph.mon.c.smithi086.stdout: persistent_cache: (nil) 2024-04-02T05:14:25.854 INFO:journalctl@ceph.mon.c.smithi086.stdout: block_size: 4096 2024-04-02T05:14:25.854 INFO:journalctl@ceph.mon.c.smithi086.stdout: block_size_deviation: 10 2024-04-02T05:14:25.854 INFO:journalctl@ceph.mon.c.smithi086.stdout: block_restart_interval: 16 2024-04-02T05:14:25.854 INFO:journalctl@ceph.mon.c.smithi086.stdout: index_block_restart_interval: 1 2024-04-02T05:14:25.854 INFO:journalctl@ceph.mon.c.smithi086.stdout: metadata_block_size: 4096 2024-04-02T05:14:25.854 INFO:journalctl@ceph.mon.c.smithi086.stdout: partition_filters: 0 2024-04-02T05:14:25.854 INFO:journalctl@ceph.mon.c.smithi086.stdout: use_delta_encoding: 1 2024-04-02T05:14:25.854 INFO:journalctl@ceph.mon.c.smithi086.stdout: filter_policy: bloomfilter 2024-04-02T05:14:25.854 INFO:journalctl@ceph.mon.c.smithi086.stdout: whole_key_filtering: 1 2024-04-02T05:14:25.854 INFO:journalctl@ceph.mon.c.smithi086.stdout: verify_compression: 0 2024-04-02T05:14:25.854 INFO:journalctl@ceph.mon.c.smithi086.stdout: read_amp_bytes_per_bit: 0 2024-04-02T05:14:25.854 INFO:journalctl@ceph.mon.c.smithi086.stdout: format_version: 5 2024-04-02T05:14:25.854 INFO:journalctl@ceph.mon.c.smithi086.stdout: enable_index_compression: 1 2024-04-02T05:14:25.855 INFO:journalctl@ceph.mon.c.smithi086.stdout: block_align: 0 2024-04-02T05:14:25.855 INFO:journalctl@ceph.mon.c.smithi086.stdout: max_auto_readahead_size: 262144 2024-04-02T05:14:25.855 INFO:journalctl@ceph.mon.c.smithi086.stdout: prepopulate_block_cache: 0 2024-04-02T05:14:25.855 INFO:journalctl@ceph.mon.c.smithi086.stdout: initial_auto_readahead_size: 8192 2024-04-02T05:14:25.855 INFO:journalctl@ceph.mon.c.smithi086.stdout: num_file_reads_for_auto_readahead: 2 2024-04-02T05:14:25.855 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.write_buffer_size: 33554432 2024-04-02T05:14:25.855 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.max_write_buffer_number: 2 2024-04-02T05:14:25.855 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.compression: NoCompression 2024-04-02T05:14:25.855 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.bottommost_compression: Disabled 2024-04-02T05:14:25.855 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.prefix_extractor: nullptr 2024-04-02T05:14:25.855 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-02T05:14:25.855 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.num_levels: 7 2024-04-02T05:14:25.855 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-02T05:14:25.856 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-02T05:14:25.856 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-02T05:14:25.856 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-02T05:14:25.856 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-02T05:14:25.856 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-02T05:14:25.856 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-02T05:14:25.856 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-02T05:14:25.856 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-02T05:14:25.856 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-02T05:14:25.856 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-02T05:14:25.856 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-02T05:14:25.856 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-02T05:14:25.856 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.compression_opts.level: 32767 2024-04-02T05:14:25.857 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.compression_opts.strategy: 0 2024-04-02T05:14:25.857 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-02T05:14:25.857 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-02T05:14:25.857 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-02T05:14:25.857 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-02T05:14:25.857 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.compression_opts.enabled: false 2024-04-02T05:14:25.857 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-02T05:14:25.857 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-02T05:14:25.857 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-02T05:14:25.857 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-02T05:14:25.857 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.target_file_size_base: 67108864 2024-04-02T05:14:25.857 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-02T05:14:25.857 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-02T05:14:25.857 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-02T05:14:25.857 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-02T05:14:25.858 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-02T05:14:25.858 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-02T05:14:25.858 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-02T05:14:25.858 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-02T05:14:25.858 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-02T05:14:25.858 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-02T05:14:25.858 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-02T05:14:25.858 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-02T05:14:25.858 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-02T05:14:25.858 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-02T05:14:25.858 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.arena_block_size: 1048576 2024-04-02T05:14:25.858 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-02T05:14:25.858 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-02T05:14:25.858 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.disable_auto_compactions: 0 2024-04-02T05:14:25.859 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-02T05:14:25.859 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-02T05:14:25.859 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-02T05:14:25.859 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-02T05:14:25.859 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-02T05:14:25.859 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-02T05:14:25.859 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-02T05:14:25.859 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-02T05:14:25.859 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-02T05:14:25.859 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-02T05:14:25.859 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.table_properties_collectors: 2024-04-02T05:14:25.859 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.inplace_update_support: 0 2024-04-02T05:14:25.859 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-02T05:14:25.859 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-02T05:14:25.859 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-02T05:14:25.860 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-02T05:14:25.860 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.bloom_locality: 0 2024-04-02T05:14:25.860 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.max_successive_merges: 0 2024-04-02T05:14:25.860 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-02T05:14:25.860 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.paranoid_file_checks: 0 2024-04-02T05:14:25.860 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.force_consistency_checks: 1 2024-04-02T05:14:25.860 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.report_bg_io_stats: 0 2024-04-02T05:14:25.860 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.ttl: 2592000 2024-04-02T05:14:25.860 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-02T05:14:25.860 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-02T05:14:25.860 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-02T05:14:25.860 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.enable_blob_files: false 2024-04-02T05:14:25.860 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.min_blob_size: 0 2024-04-02T05:14:25.860 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.blob_file_size: 268435456 2024-04-02T05:14:25.860 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-02T05:14:25.861 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-02T05:14:25.861 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-02T05:14:25.861 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-02T05:14:25.861 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-02T05:14:25.861 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.blob_file_starting_level: 0 2024-04-02T05:14:25.861 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-02T05:14:25.861 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 60.sst 2024-04-02T05:14:25.861 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-02T05:14:25.861 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 62, last_sequence is 32087, log_number is 58,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-02T05:14:25.861 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 58 2024-04-02T05:14:25.861 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: d9e61803-5eb7-49dc-978e-698a26923dc8 2024-04-02T05:14:25.861 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712034865542296, "job": 1, "event": "recovery_started", "wal_files": [58]} 2024-04-02T05:14:25.861 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #58 mode 2 2024-04-02T05:14:25.861 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712034865547763, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 63, "file_size": 1450658, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 32093, "largest_seqno": 32470, "table_properties": {"data_size": 1447811, "index_size": 1321, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 517, "raw_key_size": 4659, "raw_average_key_size": 24, "raw_value_size": 1443479, "raw_average_value_size": 7597, "num_data_blocks": 59, "num_entries": 190, "num_filter_entries": 190, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1712034865, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "d9e61803-5eb7-49dc-978e-698a26923dc8", "db_session_id": "4UN4BSUM4MBR3CS6IERM", "orig_file_number": 63, "seqno_to_time_mapping": "N/A"}} 2024-04-02T05:14:25.861 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712034865547863, "job": 1, "event": "recovery_finished"} 2024-04-02T05:14:25.862 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: [db/version_set.cc:5047] Creating manifest 65 2024-04-02T05:14:25.862 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-02T05:14:25.862 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000058.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-02T05:14:25.862 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x561cae0f4000 2024-04-02T05:14:25.862 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: DB pointer 0x561cae0de000 2024-04-02T05:14:25.862 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-02T05:14:25.862 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-02T05:14:25.862 INFO:journalctl@ceph.mon.c.smithi086.stdout: ** DB Stats ** 2024-04-02T05:14:25.862 INFO:journalctl@ceph.mon.c.smithi086.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-02T05:14:25.862 INFO:journalctl@ceph.mon.c.smithi086.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-02T05:14:25.862 INFO:journalctl@ceph.mon.c.smithi086.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-02T05:14:25.862 INFO:journalctl@ceph.mon.c.smithi086.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-02T05:14:25.862 INFO:journalctl@ceph.mon.c.smithi086.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-02T05:14:25.862 INFO:journalctl@ceph.mon.c.smithi086.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-02T05:14:25.863 INFO:journalctl@ceph.mon.c.smithi086.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-02T05:14:25.863 INFO:journalctl@ceph.mon.c.smithi086.stdout: 2024-04-02T05:14:25.863 INFO:journalctl@ceph.mon.c.smithi086.stdout: ** Compaction Stats [default] ** 2024-04-02T05:14:25.863 INFO:journalctl@ceph.mon.c.smithi086.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-02T05:14:25.863 INFO:journalctl@ceph.mon.c.smithi086.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-02T05:14:25.863 INFO:journalctl@ceph.mon.c.smithi086.stdout: L0 1/0 1.38 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 561.5 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-04-02T05:14:25.863 INFO:journalctl@ceph.mon.c.smithi086.stdout: L6 1/0 10.18 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-02T05:14:25.863 INFO:journalctl@ceph.mon.c.smithi086.stdout: Sum 2/0 11.57 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 561.5 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-04-02T05:14:25.863 INFO:journalctl@ceph.mon.c.smithi086.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 561.5 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-04-02T05:14:25.863 INFO:journalctl@ceph.mon.c.smithi086.stdout: 2024-04-02T05:14:25.863 INFO:journalctl@ceph.mon.c.smithi086.stdout: ** Compaction Stats [default] ** 2024-04-02T05:14:25.863 INFO:journalctl@ceph.mon.c.smithi086.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-02T05:14:25.863 INFO:journalctl@ceph.mon.c.smithi086.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T05:14:25.863 INFO:journalctl@ceph.mon.c.smithi086.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 561.5 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-04-02T05:14:25.864 INFO:journalctl@ceph.mon.c.smithi086.stdout: 2024-04-02T05:14:25.864 INFO:journalctl@ceph.mon.c.smithi086.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-02T05:14:25.864 INFO:journalctl@ceph.mon.c.smithi086.stdout: 2024-04-02T05:14:25.864 INFO:journalctl@ceph.mon.c.smithi086.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-02T05:14:25.864 INFO:journalctl@ceph.mon.c.smithi086.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-04-02T05:14:25.864 INFO:journalctl@ceph.mon.c.smithi086.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-02T05:14:25.864 INFO:journalctl@ceph.mon.c.smithi086.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-02T05:14:25.864 INFO:journalctl@ceph.mon.c.smithi086.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-02T05:14:25.864 INFO:journalctl@ceph.mon.c.smithi086.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-02T05:14:25.864 INFO:journalctl@ceph.mon.c.smithi086.stdout: Cumulative compaction: 0.00 GB write, 139.81 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T05:14:25.864 INFO:journalctl@ceph.mon.c.smithi086.stdout: Interval compaction: 0.00 GB write, 139.81 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T05:14:25.864 INFO:journalctl@ceph.mon.c.smithi086.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-02T05:14:25.864 INFO:journalctl@ceph.mon.c.smithi086.stdout: Block cache BinnedLRUCache@0x561cae003090#2 capacity: 512.00 MB usage: 2.05 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.6e-05 secs_since: 0 2024-04-02T05:14:25.864 INFO:journalctl@ceph.mon.c.smithi086.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.59 KB,0.000113249%) IndexBlock(1,1.45 KB,0.000277162%) Misc(1,0.00 KB,0%) 2024-04-02T05:14:25.864 INFO:journalctl@ceph.mon.c.smithi086.stdout: 2024-04-02T05:14:25.865 INFO:journalctl@ceph.mon.c.smithi086.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-02T05:14:25.865 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: starting mon.c rank 1 at public addrs [v2:172.21.15.86:3301/0,v1:172.21.15.86:6790/0] at bind addrs [v2:172.21.15.86:3301/0,v1:172.21.15.86:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 2024-04-02T05:14:25.865 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: mon.c@-1(???) e3 preinit fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 2024-04-02T05:14:25.865 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: mon.c@-1(???).mds e1 new map 2024-04-02T05:14:25.865 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: mon.c@-1(???).mds e1 print_map 2024-04-02T05:14:25.865 INFO:journalctl@ceph.mon.c.smithi086.stdout: e1 2024-04-02T05:14:25.865 INFO:journalctl@ceph.mon.c.smithi086.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-02T05:14:25.865 INFO:journalctl@ceph.mon.c.smithi086.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-02T05:14:25.865 INFO:journalctl@ceph.mon.c.smithi086.stdout: legacy client fscid: -1 2024-04-02T05:14:25.865 INFO:journalctl@ceph.mon.c.smithi086.stdout: 2024-04-02T05:14:25.865 INFO:journalctl@ceph.mon.c.smithi086.stdout: No filesystems configured 2024-04-02T05:14:25.865 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: mon.c@-1(???).osd e111 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-02T05:14:25.865 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: mon.c@-1(???).osd e111 crush map has features 432629239337189376, adjusting msgr requires 2024-04-02T05:14:25.865 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: mon.c@-1(???).osd e111 crush map has features 432629239337189376, adjusting msgr requires 2024-04-02T05:14:25.866 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: mon.c@-1(???).osd e111 crush map has features 432629239337189376, adjusting msgr requires 2024-04-02T05:14:25.866 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 ceph-mon[88839]: mon.c@-1(???).paxosservice(auth 1..28) refresh upgraded, format 0 -> 3 2024-04-02T05:14:25.866 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 bash[88819]: 8180999b3142bc0604f44ab3bc63470afcb15ad9f18f162b326a462deb1d619d 2024-04-02T05:14:25.866 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:25 smithi086 systemd[1]: Started Ceph mon.c for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:14:27.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:26 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T05:14:27.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:26 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T05:14:27.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:26 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T05:14:27.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:26 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T05:14:27.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:26 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T05:14:27.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:26 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T05:14:27.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:26 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T05:14:27.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:26 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T05:14:27.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:26 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T05:14:28.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[86443]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-02T05:14:28.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T05:14:28.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T05:14:28.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[86443]: mon.b calling monitor election 2024-04-02T05:14:28.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[86443]: mon.c calling monitor election 2024-04-02T05:14:28.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[86443]: mon.a calling monitor election 2024-04-02T05:14:28.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T05:14:28.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[86443]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-02T05:14:28.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[86443]: monmap epoch 4 2024-04-02T05:14:28.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[86443]: fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 2024-04-02T05:14:28.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[86443]: last_changed 2024-04-02T05:14:26.987722+0000 2024-04-02T05:14:28.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[86443]: created 2024-04-02T04:43:48.251359+0000 2024-04-02T05:14:28.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[86443]: min_mon_release 19 (squid) 2024-04-02T05:14:28.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[86443]: election_strategy: 1 2024-04-02T05:14:28.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[86443]: 0: [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] mon.a 2024-04-02T05:14:28.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[86443]: 1: [v2:172.21.15.86:3301/0,v1:172.21.15.86:6790/0] mon.c 2024-04-02T05:14:28.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[86443]: 2: [v2:172.21.15.192:3300/0,v1:172.21.15.192:6789/0] mon.b 2024-04-02T05:14:28.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[86443]: fsmap 2024-04-02T05:14:28.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[86443]: osdmap e111: 8 total, 8 up, 8 in 2024-04-02T05:14:28.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[86443]: mgrmap e47: y(active, since 16s), standbys: x 2024-04-02T05:14:28.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[86443]: overall HEALTH_OK 2024-04-02T05:14:28.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='' 2024-04-02T05:14:28.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:28.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[88839]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-02T05:14:28.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T05:14:28.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T05:14:28.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[88839]: mon.b calling monitor election 2024-04-02T05:14:28.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[88839]: mon.c calling monitor election 2024-04-02T05:14:28.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[88839]: mon.a calling monitor election 2024-04-02T05:14:28.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T05:14:28.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[88839]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-02T05:14:28.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[88839]: monmap epoch 4 2024-04-02T05:14:28.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[88839]: fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 2024-04-02T05:14:28.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[88839]: last_changed 2024-04-02T05:14:26.987722+0000 2024-04-02T05:14:28.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[88839]: created 2024-04-02T04:43:48.251359+0000 2024-04-02T05:14:28.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[88839]: min_mon_release 19 (squid) 2024-04-02T05:14:28.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[88839]: election_strategy: 1 2024-04-02T05:14:28.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[88839]: 0: [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] mon.a 2024-04-02T05:14:28.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[88839]: 1: [v2:172.21.15.86:3301/0,v1:172.21.15.86:6790/0] mon.c 2024-04-02T05:14:28.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[88839]: 2: [v2:172.21.15.192:3300/0,v1:172.21.15.192:6789/0] mon.b 2024-04-02T05:14:28.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[88839]: fsmap 2024-04-02T05:14:28.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[88839]: osdmap e111: 8 total, 8 up, 8 in 2024-04-02T05:14:28.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[88839]: mgrmap e47: y(active, since 16s), standbys: x 2024-04-02T05:14:28.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[88839]: overall HEALTH_OK 2024-04-02T05:14:28.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='' 2024-04-02T05:14:28.087 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:28.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:27 smithi192 ceph-mon[68000]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-02T05:14:28.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T05:14:28.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T05:14:28.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:27 smithi192 ceph-mon[68000]: mon.b calling monitor election 2024-04-02T05:14:28.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:27 smithi192 ceph-mon[68000]: mon.c calling monitor election 2024-04-02T05:14:28.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:27 smithi192 ceph-mon[68000]: mon.a calling monitor election 2024-04-02T05:14:28.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T05:14:28.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:27 smithi192 ceph-mon[68000]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-02T05:14:28.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:27 smithi192 ceph-mon[68000]: monmap epoch 4 2024-04-02T05:14:28.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:27 smithi192 ceph-mon[68000]: fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 2024-04-02T05:14:28.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:27 smithi192 ceph-mon[68000]: last_changed 2024-04-02T05:14:26.987722+0000 2024-04-02T05:14:28.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:27 smithi192 ceph-mon[68000]: created 2024-04-02T04:43:48.251359+0000 2024-04-02T05:14:28.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:27 smithi192 ceph-mon[68000]: min_mon_release 19 (squid) 2024-04-02T05:14:28.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:27 smithi192 ceph-mon[68000]: election_strategy: 1 2024-04-02T05:14:28.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:27 smithi192 ceph-mon[68000]: 0: [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] mon.a 2024-04-02T05:14:28.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:27 smithi192 ceph-mon[68000]: 1: [v2:172.21.15.86:3301/0,v1:172.21.15.86:6790/0] mon.c 2024-04-02T05:14:28.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:27 smithi192 ceph-mon[68000]: 2: [v2:172.21.15.192:3300/0,v1:172.21.15.192:6789/0] mon.b 2024-04-02T05:14:28.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:27 smithi192 ceph-mon[68000]: fsmap 2024-04-02T05:14:28.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:27 smithi192 ceph-mon[68000]: osdmap e111: 8 total, 8 up, 8 in 2024-04-02T05:14:28.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:27 smithi192 ceph-mon[68000]: mgrmap e47: y(active, since 16s), standbys: x 2024-04-02T05:14:28.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:27 smithi192 ceph-mon[68000]: overall HEALTH_OK 2024-04-02T05:14:28.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='' 2024-04-02T05:14:28.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:29.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:29 smithi192 ceph-mon[68000]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-02T05:14:29.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:29 smithi086 ceph-mon[88839]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-02T05:14:29.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:29 smithi086 ceph-mon[86443]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-02T05:14:29.701 INFO:teuthology.orchestra.run.smithi086.stdout:true 2024-04-02T05:14:30.374 INFO:teuthology.orchestra.run.smithi086.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T05:14:30.374 INFO:teuthology.orchestra.run.smithi086.stdout:alertmanager.a smithi086 *:9093,9094 running (2m) 15s ago 24m 28.1M - 0.25.0 c8568f914cd2 660ea62bb6b9 2024-04-02T05:14:30.374 INFO:teuthology.orchestra.run.smithi086.stdout:grafana.a smithi192 *:3000 running (2m) 15s ago 24m 79.2M - 9.4.12 2bacad6d85d8 259af9e63aa5 2024-04-02T05:14:30.374 INFO:teuthology.orchestra.run.smithi086.stdout:iscsi.foo.smithi086.rsbqwx smithi086 running (2m) 15s ago 23m 53.4M - 3.5 e1d6a67b021e fe728eb940f7 2024-04-02T05:14:30.374 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.x smithi192 *:8443,9283,8765 running (3m) 15s ago 28m 449M - 19.0.0-1979-g6cec793d 78f830190839 dd1e6d0f0a95 2024-04-02T05:14:30.374 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.y smithi086 *:8443,9283,8765 running (13m) 15s ago 30m 472M - 19.0.0-1979-g6cec793d 78f830190839 3dca54a018ab 2024-04-02T05:14:30.374 INFO:teuthology.orchestra.run.smithi086.stdout:mon.a smithi086 running (28s) 15s ago 30m 33.0M 2048M 19.0.0-1979-g6cec793d 78f830190839 8e478dd4da2d 2024-04-02T05:14:30.374 INFO:teuthology.orchestra.run.smithi086.stdout:mon.b smithi192 running (78s) 15s ago 29m 36.2M 2048M 19.0.0-1979-g6cec793d 78f830190839 4125e997188f 2024-04-02T05:14:30.374 INFO:teuthology.orchestra.run.smithi086.stdout:mon.c smithi086 starting - - - 2048M 2024-04-02T05:14:30.374 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.a smithi086 *:9100 running (3m) 15s ago 25m 15.6M - 1.5.0 0da6a335fe13 05a844569d7b 2024-04-02T05:14:30.374 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.b smithi192 *:9100 running (3m) 15s ago 25m 17.4M - 1.5.0 0da6a335fe13 1a7d6788d58b 2024-04-02T05:14:30.374 INFO:teuthology.orchestra.run.smithi086.stdout:osd.0 smithi086 running (28m) 15s ago 28m 57.9M 2992M 17.2.0 e1d6a67b021e 6d90c60d1c71 2024-04-02T05:14:30.375 INFO:teuthology.orchestra.run.smithi086.stdout:osd.1 smithi086 running (28m) 15s ago 28m 56.8M 2992M 17.2.0 e1d6a67b021e 203689f71603 2024-04-02T05:14:30.375 INFO:teuthology.orchestra.run.smithi086.stdout:osd.2 smithi086 running (27m) 15s ago 27m 50.3M 2992M 17.2.0 e1d6a67b021e 472c860a9af8 2024-04-02T05:14:30.375 INFO:teuthology.orchestra.run.smithi086.stdout:osd.3 smithi086 running (27m) 15s ago 27m 52.7M 2992M 17.2.0 e1d6a67b021e 327538dac44e 2024-04-02T05:14:30.375 INFO:teuthology.orchestra.run.smithi086.stdout:osd.4 smithi192 running (27m) 15s ago 27m 56.1M 3248M 17.2.0 e1d6a67b021e 6535d7919a2d 2024-04-02T05:14:30.375 INFO:teuthology.orchestra.run.smithi086.stdout:osd.5 smithi192 running (26m) 15s ago 26m 54.9M 3248M 17.2.0 e1d6a67b021e 7259249605f2 2024-04-02T05:14:30.375 INFO:teuthology.orchestra.run.smithi086.stdout:osd.6 smithi192 running (26m) 15s ago 26m 52.9M 3248M 17.2.0 e1d6a67b021e bb7d171b02a1 2024-04-02T05:14:30.375 INFO:teuthology.orchestra.run.smithi086.stdout:osd.7 smithi192 running (25m) 15s ago 25m 54.9M 3248M 17.2.0 e1d6a67b021e 95f5a0353999 2024-04-02T05:14:30.375 INFO:teuthology.orchestra.run.smithi086.stdout:prometheus.a smithi192 *:9095 running (2m) 15s ago 25m 65.1M - 2.43.0 a07b618ecd1d 8378fcdb60e3 2024-04-02T05:14:30.375 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi086.bxzxlp smithi086 *:8000 running (23m) 15s ago 23m 78.7M - 17.2.0 e1d6a67b021e 6725785fbc53 2024-04-02T05:14:30.375 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi192.pmlbot smithi192 *:8000 running (23m) 15s ago 23m 77.9M - 17.2.0 e1d6a67b021e 5ae3638f8b1c 2024-04-02T05:14:30.787 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:14:30.787 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": { 2024-04-02T05:14:30.787 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 3 2024-04-02T05:14:30.787 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:14:30.787 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": { 2024-04-02T05:14:30.787 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 2 2024-04-02T05:14:30.787 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:14:30.787 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": { 2024-04-02T05:14:30.787 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-02T05:14:30.788 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:14:30.788 INFO:teuthology.orchestra.run.smithi086.stdout: "rgw": { 2024-04-02T05:14:30.788 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-02T05:14:30.788 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:14:30.788 INFO:teuthology.orchestra.run.smithi086.stdout: "overall": { 2024-04-02T05:14:30.788 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-04-02T05:14:30.788 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 5 2024-04-02T05:14:30.788 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-04-02T05:14:30.788 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:14:30.788 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:14:30.558+0000 7fe19d012700 -1 mgr.server handle_report got status from non-daemon mon.c 2024-04-02T05:14:31.124 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:14:31.124 INFO:teuthology.orchestra.run.smithi086.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", 2024-04-02T05:14:31.124 INFO:teuthology.orchestra.run.smithi086.stdout: "in_progress": true, 2024-04-02T05:14:31.124 INFO:teuthology.orchestra.run.smithi086.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi086", 2024-04-02T05:14:31.124 INFO:teuthology.orchestra.run.smithi086.stdout: "services_complete": [], 2024-04-02T05:14:31.124 INFO:teuthology.orchestra.run.smithi086.stdout: "progress": "1/2 daemons upgraded", 2024-04-02T05:14:31.124 INFO:teuthology.orchestra.run.smithi086.stdout: "message": "Currently upgrading mon daemons", 2024-04-02T05:14:31.124 INFO:teuthology.orchestra.run.smithi086.stdout: "is_paused": false 2024-04-02T05:14:31.124 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:14:31.582 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:31 smithi086 ceph-mon[86443]: from='client.54135 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:14:31.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:31 smithi086 ceph-mon[86443]: from='client.54141 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:14:31.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:31 smithi086 ceph-mon[86443]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:14:31.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:31 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/3448003394' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:14:31.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:31 smithi086 ceph-mon[88839]: from='client.54135 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:14:31.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:31 smithi086 ceph-mon[88839]: from='client.54141 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:14:31.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:31 smithi086 ceph-mon[88839]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:14:31.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:31 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/3448003394' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:14:31.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:31 smithi192 ceph-mon[68000]: from='client.54135 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:14:31.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:31 smithi192 ceph-mon[68000]: from='client.54141 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:14:31.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:31 smithi192 ceph-mon[68000]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:14:31.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:31 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/3448003394' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:14:32.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:32 smithi086 ceph-mon[88839]: from='client.54147 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:14:32.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:32 smithi086 ceph-mon[88839]: from='client.54159 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:14:32.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:32 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:14:32.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:32 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:32.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:32 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:32.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:32 smithi086 ceph-mon[86443]: from='client.54147 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:14:32.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:32 smithi086 ceph-mon[86443]: from='client.54159 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:14:32.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:32 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:14:32.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:32 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:32.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:32 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:32.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:32 smithi192 ceph-mon[68000]: from='client.54147 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:14:32.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:32 smithi192 ceph-mon[68000]: from='client.54159 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:14:32.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:32 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:14:32.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:32 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:32.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:32 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:33.405 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:33 smithi086 ceph-mon[88839]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:14:33.405 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:33 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:33.405 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:33 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:33.405 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:33 smithi086 ceph-mon[86443]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:14:33.405 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:33 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:33.405 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:33 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:33.578 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:33 smithi192 ceph-mon[68000]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:14:33.578 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:33 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:33.579 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:33 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:33.776 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:33 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:14:33] "GET /metrics HTTP/1.1" 200 37950 "" "Prometheus/2.43.0" 2024-04-02T05:14:35.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:35 smithi086 ceph-mon[86443]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:14:35.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:35 smithi086 ceph-mon[88839]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:14:35.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:35 smithi192 ceph-mon[68000]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:14:37.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:36 smithi086 ceph-mon[88839]: Detected new or changed devices on smithi086 2024-04-02T05:14:37.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:36 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:37.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:36 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:37.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:36 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:37.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:36 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:14:37.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:36 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:37.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:36 smithi086 ceph-mon[88839]: Reconfiguring mon.a (monmap changed)... 2024-04-02T05:14:37.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:36 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T05:14:37.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:36 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T05:14:37.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:36 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:37.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:36 smithi086 ceph-mon[88839]: Reconfiguring daemon mon.a on smithi086 2024-04-02T05:14:37.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:36 smithi086 ceph-mon[88839]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:14:37.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:36 smithi086 ceph-mon[86443]: Detected new or changed devices on smithi086 2024-04-02T05:14:37.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:36 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:37.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:36 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:37.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:36 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:37.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:36 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:14:37.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:36 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:37.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:36 smithi086 ceph-mon[86443]: Reconfiguring mon.a (monmap changed)... 2024-04-02T05:14:37.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:36 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T05:14:37.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:36 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T05:14:37.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:36 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:37.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:36 smithi086 ceph-mon[86443]: Reconfiguring daemon mon.a on smithi086 2024-04-02T05:14:37.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:36 smithi086 ceph-mon[86443]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:14:37.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:36 smithi192 ceph-mon[68000]: Detected new or changed devices on smithi086 2024-04-02T05:14:37.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:36 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:37.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:36 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:37.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:36 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:37.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:36 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:14:37.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:36 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:37.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:36 smithi192 ceph-mon[68000]: Reconfiguring mon.a (monmap changed)... 2024-04-02T05:14:37.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:36 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T05:14:37.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:36 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T05:14:37.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:36 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:37.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:36 smithi192 ceph-mon[68000]: Reconfiguring daemon mon.a on smithi086 2024-04-02T05:14:37.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:36 smithi192 ceph-mon[68000]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:14:38.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:38 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:38.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:38 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:38.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:38 smithi086 ceph-mon[86443]: Reconfiguring mgr.y (monmap changed)... 2024-04-02T05:14:38.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:38 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T05:14:38.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:38 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T05:14:38.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:38 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T05:14:38.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:38 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:38.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:38 smithi086 ceph-mon[86443]: Reconfiguring daemon mgr.y on smithi086 2024-04-02T05:14:38.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:38 smithi086 ceph-mon[86443]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:14:38.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:38 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:38.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:38 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:38.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:38 smithi086 ceph-mon[88839]: Reconfiguring mgr.y (monmap changed)... 2024-04-02T05:14:38.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:38 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T05:14:38.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:38 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T05:14:38.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:38 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T05:14:38.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:38 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:38.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:38 smithi086 ceph-mon[88839]: Reconfiguring daemon mgr.y on smithi086 2024-04-02T05:14:38.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:38 smithi086 ceph-mon[88839]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:14:38.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:38 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:38.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:38 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:38.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:38 smithi192 ceph-mon[68000]: Reconfiguring mgr.y (monmap changed)... 2024-04-02T05:14:38.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:38 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T05:14:38.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:38 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T05:14:38.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:38 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T05:14:38.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:38 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:38.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:38 smithi192 ceph-mon[68000]: Reconfiguring daemon mgr.y on smithi086 2024-04-02T05:14:38.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:38 smithi192 ceph-mon[68000]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:14:40.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:39 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:40.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:39 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:40.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:39 smithi086 ceph-mon[88839]: Reconfiguring mon.c (monmap changed)... 2024-04-02T05:14:40.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:39 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T05:14:40.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:39 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T05:14:40.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:39 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:40.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:39 smithi086 ceph-mon[88839]: Reconfiguring daemon mon.c on smithi086 2024-04-02T05:14:40.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:39 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:40.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:39 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:40.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:39 smithi086 ceph-mon[86443]: Reconfiguring mon.c (monmap changed)... 2024-04-02T05:14:40.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:39 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T05:14:40.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:39 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T05:14:40.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:39 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:40.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:39 smithi086 ceph-mon[86443]: Reconfiguring daemon mon.c on smithi086 2024-04-02T05:14:40.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:39 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:40.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:39 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:40.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:39 smithi192 ceph-mon[68000]: Reconfiguring mon.c (monmap changed)... 2024-04-02T05:14:40.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:39 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T05:14:40.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:39 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T05:14:40.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:39 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:40.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:39 smithi192 ceph-mon[68000]: Reconfiguring daemon mon.c on smithi086 2024-04-02T05:14:41.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:40 smithi192 ceph-mon[68000]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:14:41.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:40 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:14:41.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:40 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:41.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:40 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:41.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:40 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-02T05:14:41.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:40 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:41.194 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:40 smithi086 ceph-mon[88839]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:14:41.194 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:40 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:14:41.194 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:40 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:41.194 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:40 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:41.194 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:40 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-02T05:14:41.194 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:40 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:41.194 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:40 smithi086 ceph-mon[86443]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:14:41.194 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:40 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:14:41.194 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:40 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:41.194 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:40 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:41.195 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:40 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-02T05:14:41.195 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:40 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:42.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:41 smithi192 ceph-mon[68000]: Reconfiguring osd.0 (monmap changed)... 2024-04-02T05:14:42.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:41 smithi192 ceph-mon[68000]: Reconfiguring daemon osd.0 on smithi086 2024-04-02T05:14:42.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:41 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:14:42.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:41 smithi086 ceph-mon[86443]: Reconfiguring osd.0 (monmap changed)... 2024-04-02T05:14:42.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:41 smithi086 ceph-mon[86443]: Reconfiguring daemon osd.0 on smithi086 2024-04-02T05:14:42.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:41 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:14:42.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:41 smithi086 ceph-mon[88839]: Reconfiguring osd.0 (monmap changed)... 2024-04-02T05:14:42.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:41 smithi086 ceph-mon[88839]: Reconfiguring daemon osd.0 on smithi086 2024-04-02T05:14:42.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:41 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:14:43.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:43 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:43.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:43 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:43.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:43 smithi086 ceph-mon[88839]: Reconfiguring osd.1 (monmap changed)... 2024-04-02T05:14:43.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:43 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-02T05:14:43.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:43 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:43.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:43 smithi086 ceph-mon[88839]: Reconfiguring daemon osd.1 on smithi086 2024-04-02T05:14:43.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:43 smithi086 ceph-mon[88839]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:14:43.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:43 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:43.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:43 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:43.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:43 smithi086 ceph-mon[86443]: Reconfiguring osd.1 (monmap changed)... 2024-04-02T05:14:43.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:43 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-02T05:14:43.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:43 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:43.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:43 smithi086 ceph-mon[86443]: Reconfiguring daemon osd.1 on smithi086 2024-04-02T05:14:43.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:43 smithi086 ceph-mon[86443]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:14:43.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:43 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:43.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:43 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:43.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:43 smithi192 ceph-mon[68000]: Reconfiguring osd.1 (monmap changed)... 2024-04-02T05:14:43.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:43 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-02T05:14:43.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:43 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:43.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:43 smithi192 ceph-mon[68000]: Reconfiguring daemon osd.1 on smithi086 2024-04-02T05:14:43.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:43 smithi192 ceph-mon[68000]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:14:43.722 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:43 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:14:43] "GET /metrics HTTP/1.1" 200 38016 "" "Prometheus/2.43.0" 2024-04-02T05:14:45.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:44 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:45.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:44 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:45.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:44 smithi086 ceph-mon[88839]: Reconfiguring osd.2 (monmap changed)... 2024-04-02T05:14:45.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:44 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-02T05:14:45.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:44 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:45.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:44 smithi086 ceph-mon[88839]: Reconfiguring daemon osd.2 on smithi086 2024-04-02T05:14:45.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:44 smithi086 ceph-mon[88839]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:14:45.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:44 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:45.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:44 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:45.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:44 smithi086 ceph-mon[86443]: Reconfiguring osd.2 (monmap changed)... 2024-04-02T05:14:45.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:44 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-02T05:14:45.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:44 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:45.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:44 smithi086 ceph-mon[86443]: Reconfiguring daemon osd.2 on smithi086 2024-04-02T05:14:45.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:44 smithi086 ceph-mon[86443]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:14:45.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:44 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:45.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:44 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:45.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:44 smithi192 ceph-mon[68000]: Reconfiguring osd.2 (monmap changed)... 2024-04-02T05:14:45.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:44 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-02T05:14:45.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:44 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:45.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:44 smithi192 ceph-mon[68000]: Reconfiguring daemon osd.2 on smithi086 2024-04-02T05:14:45.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:44 smithi192 ceph-mon[68000]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:14:46.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:46 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:46.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:46 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:46.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:46 smithi086 ceph-mon[86443]: Reconfiguring osd.3 (monmap changed)... 2024-04-02T05:14:46.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:46 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-02T05:14:46.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:46 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:46.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:46 smithi086 ceph-mon[86443]: Reconfiguring daemon osd.3 on smithi086 2024-04-02T05:14:46.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:46 smithi086 ceph-mon[86443]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:14:46.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:46 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:46.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:46 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:46.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:46 smithi086 ceph-mon[88839]: Reconfiguring osd.3 (monmap changed)... 2024-04-02T05:14:46.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:46 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-02T05:14:46.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:46 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:46.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:46 smithi086 ceph-mon[88839]: Reconfiguring daemon osd.3 on smithi086 2024-04-02T05:14:46.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:46 smithi086 ceph-mon[88839]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:14:46.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:46 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:46.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:46 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:46.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:46 smithi192 ceph-mon[68000]: Reconfiguring osd.3 (monmap changed)... 2024-04-02T05:14:46.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:46 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-02T05:14:46.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:46 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:46.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:46 smithi192 ceph-mon[68000]: Reconfiguring daemon osd.3 on smithi086 2024-04-02T05:14:46.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:46 smithi192 ceph-mon[68000]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:14:48.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:48 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:48.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:48 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:48.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:48 smithi086 ceph-mon[88839]: Reconfiguring rgw.foo.smithi086.bxzxlp (monmap changed)... 2024-04-02T05:14:48.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:48 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi086.bxzxlp", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T05:14:48.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:48 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi086.bxzxlp", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T05:14:48.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:48 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:48.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:48 smithi086 ceph-mon[88839]: Reconfiguring daemon rgw.foo.smithi086.bxzxlp on smithi086 2024-04-02T05:14:48.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:48 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:48.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:48 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:48.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:48 smithi086 ceph-mon[86443]: Reconfiguring rgw.foo.smithi086.bxzxlp (monmap changed)... 2024-04-02T05:14:48.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:48 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi086.bxzxlp", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T05:14:48.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:48 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi086.bxzxlp", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T05:14:48.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:48 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:48.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:48 smithi086 ceph-mon[86443]: Reconfiguring daemon rgw.foo.smithi086.bxzxlp on smithi086 2024-04-02T05:14:48.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:48 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:48.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:48 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:48.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:48 smithi192 ceph-mon[68000]: Reconfiguring rgw.foo.smithi086.bxzxlp (monmap changed)... 2024-04-02T05:14:48.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:48 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi086.bxzxlp", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T05:14:48.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:48 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi086.bxzxlp", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T05:14:48.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:48 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:48.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:48 smithi192 ceph-mon[68000]: Reconfiguring daemon rgw.foo.smithi086.bxzxlp on smithi086 2024-04-02T05:14:49.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:49 smithi192 ceph-mon[68000]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:14:49.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:49 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:49.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:49 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:49.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:49 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T05:14:49.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:49 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T05:14:49.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:49 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:49.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:49 smithi086 ceph-mon[86443]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:14:49.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:49 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:49.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:49 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:49.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:49 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T05:14:49.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:49 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T05:14:49.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:49 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:49.586 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:49 smithi086 ceph-mon[88839]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:14:49.586 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:49 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:49.586 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:49 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:49.586 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:49 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T05:14:49.586 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:49 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T05:14:49.586 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:49 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:50.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:50 smithi192 ceph-mon[68000]: Reconfiguring mon.b (monmap changed)... 2024-04-02T05:14:50.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:50 smithi192 ceph-mon[68000]: Reconfiguring daemon mon.b on smithi192 2024-04-02T05:14:50.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:50 smithi086 ceph-mon[86443]: Reconfiguring mon.b (monmap changed)... 2024-04-02T05:14:50.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:50 smithi086 ceph-mon[86443]: Reconfiguring daemon mon.b on smithi192 2024-04-02T05:14:50.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:50 smithi086 ceph-mon[88839]: Reconfiguring mon.b (monmap changed)... 2024-04-02T05:14:50.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:50 smithi086 ceph-mon[88839]: Reconfiguring daemon mon.b on smithi192 2024-04-02T05:14:51.539 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:51 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:51.539 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:51 smithi192 ceph-mon[68000]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:14:51.539 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:51 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:51.539 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:51 smithi192 ceph-mon[68000]: Reconfiguring mgr.x (monmap changed)... 2024-04-02T05:14:51.539 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:51 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T05:14:51.540 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:51 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T05:14:51.540 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:51 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T05:14:51.540 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:51 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:51.540 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:51 smithi192 ceph-mon[68000]: Reconfiguring daemon mgr.x on smithi192 2024-04-02T05:14:51.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:51 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:51.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:51 smithi086 ceph-mon[86443]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:14:51.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:51 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:51.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:51 smithi086 ceph-mon[86443]: Reconfiguring mgr.x (monmap changed)... 2024-04-02T05:14:51.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:51 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T05:14:51.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:51 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T05:14:51.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:51 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T05:14:51.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:51 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:51.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:51 smithi086 ceph-mon[86443]: Reconfiguring daemon mgr.x on smithi192 2024-04-02T05:14:51.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:51 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:51.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:51 smithi086 ceph-mon[88839]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:14:51.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:51 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:51.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:51 smithi086 ceph-mon[88839]: Reconfiguring mgr.x (monmap changed)... 2024-04-02T05:14:51.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:51 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T05:14:51.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:51 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T05:14:51.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:51 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T05:14:51.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:51 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:51.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:51 smithi086 ceph-mon[88839]: Reconfiguring daemon mgr.x on smithi192 2024-04-02T05:14:52.576 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:52 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:14:52.576 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:52 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:52.576 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:52 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:52.576 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:52 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-02T05:14:52.576 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:52 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:52.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:52 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:14:52.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:52 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:52.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:52 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:52.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:52 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-02T05:14:52.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:52 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:52.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:52 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:14:52.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:52 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:52.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:52 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:52.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:52 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-02T05:14:52.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:52 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:53.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:14:53 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:14:53] "GET /metrics HTTP/1.1" 200 38016 "" "Prometheus/2.43.0" 2024-04-02T05:14:53.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:53 smithi086 ceph-mon[86443]: Reconfiguring osd.4 (monmap changed)... 2024-04-02T05:14:53.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:53 smithi086 ceph-mon[86443]: Reconfiguring daemon osd.4 on smithi192 2024-04-02T05:14:53.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:53 smithi086 ceph-mon[86443]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:14:53.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:53 smithi086 ceph-mon[88839]: Reconfiguring osd.4 (monmap changed)... 2024-04-02T05:14:53.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:53 smithi086 ceph-mon[88839]: Reconfiguring daemon osd.4 on smithi192 2024-04-02T05:14:53.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:53 smithi086 ceph-mon[88839]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:14:53.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:53 smithi192 ceph-mon[68000]: Reconfiguring osd.4 (monmap changed)... 2024-04-02T05:14:53.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:53 smithi192 ceph-mon[68000]: Reconfiguring daemon osd.4 on smithi192 2024-04-02T05:14:53.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:53 smithi192 ceph-mon[68000]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:14:54.646 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:54 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:54.646 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:54 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:54.646 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:54 smithi086 ceph-mon[88839]: Reconfiguring osd.5 (monmap changed)... 2024-04-02T05:14:54.646 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:54 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-02T05:14:54.646 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:54 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:54.647 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:54 smithi086 ceph-mon[88839]: Reconfiguring daemon osd.5 on smithi192 2024-04-02T05:14:54.647 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:54 smithi086 ceph-mon[88839]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:14:54.647 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:54 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:54.647 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:54 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:54.647 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:54 smithi086 ceph-mon[86443]: Reconfiguring osd.5 (monmap changed)... 2024-04-02T05:14:54.647 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:54 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-02T05:14:54.647 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:54 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:54.647 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:54 smithi086 ceph-mon[86443]: Reconfiguring daemon osd.5 on smithi192 2024-04-02T05:14:54.647 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:54 smithi086 ceph-mon[86443]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:14:54.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:54 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:54.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:54 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:54.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:54 smithi192 ceph-mon[68000]: Reconfiguring osd.5 (monmap changed)... 2024-04-02T05:14:54.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:54 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-02T05:14:54.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:54 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:54.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:54 smithi192 ceph-mon[68000]: Reconfiguring daemon osd.5 on smithi192 2024-04-02T05:14:54.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:54 smithi192 ceph-mon[68000]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:14:56.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:56 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:56.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:56 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:56.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:56 smithi086 ceph-mon[88839]: Reconfiguring osd.6 (monmap changed)... 2024-04-02T05:14:56.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:56 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-02T05:14:56.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:56 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:56.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:56 smithi086 ceph-mon[88839]: Reconfiguring daemon osd.6 on smithi192 2024-04-02T05:14:56.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:56 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:14:56.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:56 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:56.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:56 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:56.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:56 smithi086 ceph-mon[86443]: Reconfiguring osd.6 (monmap changed)... 2024-04-02T05:14:56.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:56 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-02T05:14:56.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:56 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:56.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:56 smithi086 ceph-mon[86443]: Reconfiguring daemon osd.6 on smithi192 2024-04-02T05:14:56.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:56 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:14:56.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:56 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:56.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:56 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:56.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:56 smithi192 ceph-mon[68000]: Reconfiguring osd.6 (monmap changed)... 2024-04-02T05:14:56.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:56 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-02T05:14:56.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:56 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:56.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:56 smithi192 ceph-mon[68000]: Reconfiguring daemon osd.6 on smithi192 2024-04-02T05:14:56.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:56 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:14:57.341 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:57 smithi086 ceph-mon[86443]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:14:57.341 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:57 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:57.341 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:57 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:57.341 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:57 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-02T05:14:57.341 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:57 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:57.342 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:57 smithi086 ceph-mon[88839]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:14:57.342 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:57 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:57.342 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:57 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:57.342 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:57 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-02T05:14:57.342 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:57 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:57.394 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:57 smithi192 ceph-mon[68000]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:14:57.394 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:57 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:57.394 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:57 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:57.394 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:57 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-02T05:14:57.394 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:57 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:58.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:58 smithi086 ceph-mon[86443]: Reconfiguring osd.7 (monmap changed)... 2024-04-02T05:14:58.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:58 smithi086 ceph-mon[86443]: Reconfiguring daemon osd.7 on smithi192 2024-04-02T05:14:58.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:58 smithi086 ceph-mon[88839]: Reconfiguring osd.7 (monmap changed)... 2024-04-02T05:14:58.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:58 smithi086 ceph-mon[88839]: Reconfiguring daemon osd.7 on smithi192 2024-04-02T05:14:58.376 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:58 smithi192 ceph-mon[68000]: Reconfiguring osd.7 (monmap changed)... 2024-04-02T05:14:58.376 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:58 smithi192 ceph-mon[68000]: Reconfiguring daemon osd.7 on smithi192 2024-04-02T05:14:59.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:59 smithi086 ceph-mon[88839]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:14:59.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:59 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:59.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:59 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:59.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:59 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi192.pmlbot", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T05:14:59.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:59 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi192.pmlbot", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T05:14:59.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:14:59 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:59.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:59 smithi086 ceph-mon[86443]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:14:59.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:59 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:59.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:59 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:59.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:59 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi192.pmlbot", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T05:14:59.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:59 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi192.pmlbot", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T05:14:59.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:14:59 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:14:59.396 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:59 smithi192 ceph-mon[68000]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:14:59.396 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:59 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:59.396 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:59 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:14:59.396 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:59 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi192.pmlbot", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T05:14:59.396 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:59 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi192.pmlbot", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T05:14:59.396 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:14:59 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:15:00.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:00 smithi086 ceph-mon[88839]: Reconfiguring rgw.foo.smithi192.pmlbot (monmap changed)... 2024-04-02T05:15:00.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:00 smithi086 ceph-mon[88839]: Reconfiguring daemon rgw.foo.smithi192.pmlbot on smithi192 2024-04-02T05:15:00.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:00 smithi086 ceph-mon[86443]: Reconfiguring rgw.foo.smithi192.pmlbot (monmap changed)... 2024-04-02T05:15:00.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:00 smithi086 ceph-mon[86443]: Reconfiguring daemon rgw.foo.smithi192.pmlbot on smithi192 2024-04-02T05:15:00.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:00 smithi192 ceph-mon[68000]: Reconfiguring rgw.foo.smithi192.pmlbot (monmap changed)... 2024-04-02T05:15:00.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:00 smithi192 ceph-mon[68000]: Reconfiguring daemon rgw.foo.smithi192.pmlbot on smithi192 2024-04-02T05:15:01.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:01.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:01.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:15:01.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all mon 2024-04-02T05:15:01.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:01.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-02T05:15:01.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-02T05:15:01.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all crash 2024-04-02T05:15:01.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:01.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all mds 2024-04-02T05:15:01.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:01.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all rbd-mirror 2024-04-02T05:15:01.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:01.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-02T05:15:01.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all ceph-exporter 2024-04-02T05:15:01.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:01.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all nfs 2024-04-02T05:15:01.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:01.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all nvmeof 2024-04-02T05:15:01.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:01.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.430 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all node-exporter 2024-04-02T05:15:01.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all prometheus 2024-04-02T05:15:01.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all alertmanager 2024-04-02T05:15:01.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.431 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all grafana 2024-04-02T05:15:01.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.432 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all loki 2024-04-02T05:15:01.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all promtail 2024-04-02T05:15:01.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: Upgrade: Finalizing container_image settings 2024-04-02T05:15:01.433 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:15:01.434 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:15:01.434 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-02T05:15:01.434 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.434 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.434 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-02T05:15:01.434 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:15:01.434 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:15:01.434 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-02T05:15:01.434 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:15:01.434 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:15:01.434 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:15:01.434 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:15:01.434 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-02T05:15:01.434 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:15:01.435 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:15:01.435 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:15:01.435 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:15:01.435 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-02T05:15:01.435 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:15:01.435 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.435 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.435 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:15:01.435 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:15:01.435 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-02T05:15:01.435 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:15:01.435 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:15:01.435 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:15:01.435 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:15:01.435 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-02T05:15:01.436 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:15:01.436 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:15:01.436 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-02T05:15:01.436 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.436 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.436 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.436 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.436 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.436 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.436 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.436 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.436 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.436 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.436 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.437 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.437 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: Upgrade: Complete! 2024-04-02T05:15:01.437 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:15:01.437 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:15:01.437 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-02T05:15:01.437 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:15:01.437 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:15:01.437 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:01.437 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:01.437 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:15:01.437 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:15:01.437 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:01.477 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:01.477 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:01.478 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:15:01.478 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.478 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.478 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all mon 2024-04-02T05:15:01.478 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:01.478 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.478 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.478 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-02T05:15:01.478 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.478 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.478 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-02T05:15:01.478 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.479 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all crash 2024-04-02T05:15:01.479 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:01.479 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.479 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all mds 2024-04-02T05:15:01.479 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:01.479 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.479 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all rbd-mirror 2024-04-02T05:15:01.479 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:01.479 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.479 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-02T05:15:01.479 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.479 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.479 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.479 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.479 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.480 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all ceph-exporter 2024-04-02T05:15:01.480 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:01.480 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.480 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:01.480 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:01.480 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:15:01.480 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.480 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.480 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all mon 2024-04-02T05:15:01.480 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:01.481 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.481 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.481 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-02T05:15:01.481 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.481 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.481 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-02T05:15:01.481 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.481 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all crash 2024-04-02T05:15:01.481 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:01.481 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.481 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all mds 2024-04-02T05:15:01.481 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:01.481 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.481 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all rbd-mirror 2024-04-02T05:15:01.482 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:01.482 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.482 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-02T05:15:01.482 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.482 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.482 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.482 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.482 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.482 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all ceph-exporter 2024-04-02T05:15:01.482 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:01.482 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.483 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all nfs 2024-04-02T05:15:01.483 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:01.483 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.483 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all nvmeof 2024-04-02T05:15:01.483 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:01.483 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.483 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all node-exporter 2024-04-02T05:15:01.483 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.483 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.483 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.483 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.483 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.483 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all prometheus 2024-04-02T05:15:01.483 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.484 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.484 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.484 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.484 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.484 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all alertmanager 2024-04-02T05:15:01.484 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.484 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.484 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.484 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.484 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.484 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all grafana 2024-04-02T05:15:01.484 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.484 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.484 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.485 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all nfs 2024-04-02T05:15:01.485 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:01.485 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.485 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all nvmeof 2024-04-02T05:15:01.485 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:01.485 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.485 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all node-exporter 2024-04-02T05:15:01.485 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.485 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.485 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.485 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.485 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.485 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all prometheus 2024-04-02T05:15:01.486 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.486 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.486 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.486 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.486 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.486 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all alertmanager 2024-04-02T05:15:01.486 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.486 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.486 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.486 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.486 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.486 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all grafana 2024-04-02T05:15:01.486 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.487 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.487 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.487 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.487 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.487 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all loki 2024-04-02T05:15:01.487 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.487 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.487 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.487 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.487 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.487 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all promtail 2024-04-02T05:15:01.488 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.488 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.488 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.488 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.488 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: Upgrade: Finalizing container_image settings 2024-04-02T05:15:01.488 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:15:01.488 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:15:01.488 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-02T05:15:01.488 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.488 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.488 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-02T05:15:01.488 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:15:01.488 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:15:01.488 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-02T05:15:01.488 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:15:01.489 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:15:01.489 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:15:01.489 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.489 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.489 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all loki 2024-04-02T05:15:01.489 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.489 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.489 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.489 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.489 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:01.489 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all promtail 2024-04-02T05:15:01.489 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.490 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-02T05:15:01.490 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.490 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-02T05:15:01.490 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: Upgrade: Finalizing container_image settings 2024-04-02T05:15:01.490 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:15:01.490 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:15:01.490 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-02T05:15:01.490 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.490 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.490 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-02T05:15:01.490 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:15:01.490 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:15:01.490 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-02T05:15:01.490 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:15:01.491 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:15:01.491 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:15:01.491 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:15:01.491 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-02T05:15:01.491 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:15:01.491 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:15:01.491 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:15:01.491 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:15:01.491 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-02T05:15:01.491 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:15:01.491 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.491 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.492 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:15:01.492 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:15:01.492 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-02T05:15:01.492 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:15:01.492 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:15:01.492 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:15:01.492 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:15:01.492 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-02T05:15:01.492 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:15:01.492 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:15:01.492 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-02T05:15:01.492 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.492 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.492 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.493 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:15:01.493 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-02T05:15:01.493 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:15:01.493 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:15:01.493 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:15:01.493 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:15:01.493 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-02T05:15:01.493 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:15:01.493 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.493 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.493 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:15:01.494 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:15:01.494 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-02T05:15:01.494 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:15:01.494 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:15:01.494 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:15:01.494 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:15:01.494 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-02T05:15:01.494 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:15:01.494 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:15:01.494 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-02T05:15:01.494 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.494 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.494 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.495 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.495 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.495 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.495 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.495 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.495 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.495 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.495 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.495 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.495 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: Upgrade: Complete! 2024-04-02T05:15:01.495 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:15:01.495 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:15:01.496 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-02T05:15:01.496 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:15:01.496 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:15:01.496 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:01.496 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:01.496 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:15:01.496 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:15:01.496 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:01.496 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.496 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.496 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.496 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.496 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.497 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.497 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.497 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.497 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:15:01.497 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: Upgrade: Complete! 2024-04-02T05:15:01.497 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:15:01.497 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:15:01.497 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-02T05:15:01.497 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:15:01.497 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:15:01.497 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:01.497 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:01.497 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:15:01.497 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:15:01.497 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:02.035 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch ps' 2024-04-02T05:15:02.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:02 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:15:02.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:02 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:15:02.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:02 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:15:03.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:03 smithi086 ceph-mon[86443]: from='client.64108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:15:03.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:03 smithi086 ceph-mon[86443]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:15:03.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:03 smithi086 ceph-mon[88839]: from='client.64108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:15:03.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:03 smithi086 ceph-mon[88839]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:15:03.578 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:03 smithi192 ceph-mon[68000]: from='client.64108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:15:03.578 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:03 smithi192 ceph-mon[68000]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:15:03.644 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:15:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:15:03] "GET /metrics HTTP/1.1" 200 38014 "" "Prometheus/2.43.0" 2024-04-02T05:15:04.114 INFO:teuthology.orchestra.run.smithi086.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T05:15:04.114 INFO:teuthology.orchestra.run.smithi086.stdout:alertmanager.a smithi086 *:9093,9094 running (3m) 32s ago 25m 28.1M - 0.25.0 c8568f914cd2 660ea62bb6b9 2024-04-02T05:15:04.114 INFO:teuthology.orchestra.run.smithi086.stdout:grafana.a smithi192 *:3000 running (2m) 49s ago 24m 79.2M - 9.4.12 2bacad6d85d8 259af9e63aa5 2024-04-02T05:15:04.114 INFO:teuthology.orchestra.run.smithi086.stdout:iscsi.foo.smithi086.rsbqwx smithi086 running (2m) 32s ago 24m 56.3M - 3.5 e1d6a67b021e fe728eb940f7 2024-04-02T05:15:04.115 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.x smithi192 *:8443,9283,8765 running (4m) 49s ago 29m 449M - 19.0.0-1979-g6cec793d 78f830190839 dd1e6d0f0a95 2024-04-02T05:15:04.115 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.y smithi086 *:8443,9283,8765 running (13m) 32s ago 31m 492M - 19.0.0-1979-g6cec793d 78f830190839 3dca54a018ab 2024-04-02T05:15:04.115 INFO:teuthology.orchestra.run.smithi086.stdout:mon.a smithi086 running (62s) 32s ago 31m 41.0M 2048M 19.0.0-1979-g6cec793d 78f830190839 8e478dd4da2d 2024-04-02T05:15:04.115 INFO:teuthology.orchestra.run.smithi086.stdout:mon.b smithi192 running (112s) 49s ago 29m 36.2M 2048M 19.0.0-1979-g6cec793d 78f830190839 4125e997188f 2024-04-02T05:15:04.115 INFO:teuthology.orchestra.run.smithi086.stdout:mon.c smithi086 running (38s) 32s ago 29m 22.3M 2048M 19.0.0-1979-g6cec793d 78f830190839 8180999b3142 2024-04-02T05:15:04.115 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.a smithi086 *:9100 running (3m) 32s ago 25m 15.5M - 1.5.0 0da6a335fe13 05a844569d7b 2024-04-02T05:15:04.115 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.b smithi192 *:9100 running (3m) 49s ago 25m 17.4M - 1.5.0 0da6a335fe13 1a7d6788d58b 2024-04-02T05:15:04.115 INFO:teuthology.orchestra.run.smithi086.stdout:osd.0 smithi086 running (29m) 32s ago 29m 58.1M 2992M 17.2.0 e1d6a67b021e 6d90c60d1c71 2024-04-02T05:15:04.115 INFO:teuthology.orchestra.run.smithi086.stdout:osd.1 smithi086 running (28m) 32s ago 28m 56.8M 2992M 17.2.0 e1d6a67b021e 203689f71603 2024-04-02T05:15:04.115 INFO:teuthology.orchestra.run.smithi086.stdout:osd.2 smithi086 running (28m) 32s ago 28m 50.7M 2992M 17.2.0 e1d6a67b021e 472c860a9af8 2024-04-02T05:15:04.115 INFO:teuthology.orchestra.run.smithi086.stdout:osd.3 smithi086 running (27m) 32s ago 27m 52.9M 2992M 17.2.0 e1d6a67b021e 327538dac44e 2024-04-02T05:15:04.115 INFO:teuthology.orchestra.run.smithi086.stdout:osd.4 smithi192 running (27m) 49s ago 27m 56.1M 3248M 17.2.0 e1d6a67b021e 6535d7919a2d 2024-04-02T05:15:04.115 INFO:teuthology.orchestra.run.smithi086.stdout:osd.5 smithi192 running (27m) 49s ago 27m 54.9M 3248M 17.2.0 e1d6a67b021e 7259249605f2 2024-04-02T05:15:04.115 INFO:teuthology.orchestra.run.smithi086.stdout:osd.6 smithi192 running (26m) 49s ago 26m 52.9M 3248M 17.2.0 e1d6a67b021e bb7d171b02a1 2024-04-02T05:15:04.116 INFO:teuthology.orchestra.run.smithi086.stdout:osd.7 smithi192 running (26m) 49s ago 26m 54.9M 3248M 17.2.0 e1d6a67b021e 95f5a0353999 2024-04-02T05:15:04.116 INFO:teuthology.orchestra.run.smithi086.stdout:prometheus.a smithi192 *:9095 running (3m) 49s ago 25m 65.1M - 2.43.0 a07b618ecd1d 8378fcdb60e3 2024-04-02T05:15:04.116 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi086.bxzxlp smithi086 *:8000 running (24m) 32s ago 24m 79.0M - 17.2.0 e1d6a67b021e 6725785fbc53 2024-04-02T05:15:04.116 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi192.pmlbot smithi192 *:8000 running (24m) 49s ago 24m 77.9M - 17.2.0 e1d6a67b021e 5ae3638f8b1c 2024-04-02T05:15:04.798 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 1'"'"'' 2024-04-02T05:15:05.393 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:05 smithi086 ceph-mon[88839]: from='client.54171 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:15:05.394 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:05 smithi086 ceph-mon[88839]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:15:05.394 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:05 smithi086 ceph-mon[86443]: from='client.54171 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:15:05.394 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:05 smithi086 ceph-mon[86443]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:15:05.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:05 smithi192 ceph-mon[68000]: from='client.54171 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:15:05.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:05 smithi192 ceph-mon[68000]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:15:06.856 INFO:teuthology.orchestra.run.smithi086.stdout:true 2024-04-02T05:15:07.546 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:07 smithi086 ceph-mon[88839]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:15:07.546 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:07 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/3309680688' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:07.546 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:07 smithi086 ceph-mon[86443]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:15:07.546 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:07 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/3309680688' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:07.571 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph versions | jq -e '"'"'.mon | keys'"'"' | grep $sha1' 2024-04-02T05:15:07.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:07 smithi192 ceph-mon[68000]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:15:07.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:07 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/3309680688' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:09.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:09 smithi086 ceph-mon[88839]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:15:09.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:09 smithi086 ceph-mon[86443]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:15:09.570 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)" 2024-04-02T05:15:09.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:09 smithi192 ceph-mon[68000]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:15:10.157 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 5'"'"'' 2024-04-02T05:15:10.459 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:10 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/3095900570' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:10.460 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:10 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/3095900570' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:10.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:10 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/3095900570' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:11.531 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:11 smithi086 ceph-mon[86443]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:15:11.532 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:11 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:15:11.532 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:11 smithi086 ceph-mon[88839]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:15:11.532 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:11 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:15:11.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:11 smithi192 ceph-mon[68000]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:15:11.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:11 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:15:12.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:12 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:15:12.631 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:12 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:15:12.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:12 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:15:13.403 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:13 smithi086 ceph-mon[86443]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:15:13.404 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:13 smithi086 ceph-mon[86443]: from='client.64132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:15:13.404 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:13 smithi086 ceph-mon[88839]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:15:13.404 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:13 smithi086 ceph-mon[88839]: from='client.64132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:15:13.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:13 smithi192 ceph-mon[68000]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:15:13.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:13 smithi192 ceph-mon[68000]: from='client.64132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:15:13.832 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:15:13 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:15:13] "GET /metrics HTTP/1.1" 200 38018 "" "Prometheus/2.43.0" 2024-04-02T05:15:13.893 INFO:teuthology.orchestra.run.smithi086.stdout:true 2024-04-02T05:15:14.542 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch upgrade status' 2024-04-02T05:15:15.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:15 smithi086 ceph-mon[86443]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:15:15.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:15 smithi086 ceph-mon[88839]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:15:15.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:15 smithi192 ceph-mon[68000]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:15:16.445 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:15:16.445 INFO:teuthology.orchestra.run.smithi086.stdout: "target_image": null, 2024-04-02T05:15:16.445 INFO:teuthology.orchestra.run.smithi086.stdout: "in_progress": false, 2024-04-02T05:15:16.445 INFO:teuthology.orchestra.run.smithi086.stdout: "which": "", 2024-04-02T05:15:16.445 INFO:teuthology.orchestra.run.smithi086.stdout: "services_complete": [], 2024-04-02T05:15:16.445 INFO:teuthology.orchestra.run.smithi086.stdout: "progress": null, 2024-04-02T05:15:16.445 INFO:teuthology.orchestra.run.smithi086.stdout: "message": "", 2024-04-02T05:15:16.446 INFO:teuthology.orchestra.run.smithi086.stdout: "is_paused": false 2024-04-02T05:15:16.446 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:15:17.118 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph health detail' 2024-04-02T05:15:17.424 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:17 smithi086 ceph-mon[86443]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:15:17.425 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:17 smithi086 ceph-mon[88839]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:15:17.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:17 smithi192 ceph-mon[68000]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:15:18.483 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:18 smithi086 ceph-mon[88839]: from='client.64138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:15:18.483 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:18 smithi086 ceph-mon[86443]: from='client.64138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:15:18.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:18 smithi192 ceph-mon[68000]: from='client.64138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:15:19.211 INFO:teuthology.orchestra.run.smithi086.stdout:HEALTH_OK 2024-04-02T05:15:19.548 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:19 smithi086 ceph-mon[88839]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:15:19.548 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:19 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/780582688' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T05:15:19.548 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:19 smithi086 ceph-mon[86443]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:15:19.548 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:19 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/780582688' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T05:15:19.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:19 smithi192 ceph-mon[68000]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:15:19.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:19 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/780582688' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T05:15:19.960 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2' 2024-04-02T05:15:21.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:21 smithi086 ceph-mon[88839]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:15:21.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:21 smithi086 ceph-mon[86443]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:15:21.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:21 smithi192 ceph-mon[68000]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:15:22.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:22 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:15:22.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:22 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:15:22.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:22 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:15:23.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:23 smithi086 ceph-mon[88839]: from='client.54207 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:15:23.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:23 smithi086 ceph-mon[88839]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:15:23.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:23 smithi086 ceph-mon[86443]: from='client.54207 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:15:23.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:23 smithi086 ceph-mon[86443]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:15:23.586 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:15:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:15:23] "GET /metrics HTTP/1.1" 200 38018 "" "Prometheus/2.43.0" 2024-04-02T05:15:23.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:23 smithi192 ceph-mon[68000]: from='client.54207 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:15:23.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:23 smithi192 ceph-mon[68000]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:15:23.810 INFO:teuthology.orchestra.run.smithi086.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:15:24.467 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-02T05:15:25.025 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:24 smithi086 ceph-mon[88839]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:15:25.025 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:24 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:25.025 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:24 smithi086 ceph-mon[88839]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:15:25.025 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:24 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:25.025 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:24 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:25.025 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:24 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:15:25.025 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:24 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:15:25.025 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:24 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:25.025 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:24 smithi086 ceph-mon[86443]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:15:25.025 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:24 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:25.026 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:24 smithi086 ceph-mon[86443]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:15:25.026 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:24 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:25.026 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:24 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:25.026 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:24 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:15:25.026 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:24 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:15:25.026 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:24 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:25.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:24 smithi192 ceph-mon[68000]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:15:25.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:24 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:25.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:24 smithi192 ceph-mon[68000]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:15:25.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:24 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:25.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:24 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:25.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:24 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:15:25.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:24 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:15:25.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:24 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:26.060 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:25 smithi086 ceph-mon[88839]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:15:26.060 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:25 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:15:26.061 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:25 smithi086 ceph-mon[86443]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:15:26.061 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:25 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:15:26.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:25 smithi192 ceph-mon[68000]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:15:26.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:25 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:15:26.835 INFO:teuthology.orchestra.run.smithi086.stdout:true 2024-04-02T05:15:27.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:26 smithi086 ceph-mon[88839]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:15:27.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:26 smithi086 ceph-mon[86443]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:15:27.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:26 smithi192 ceph-mon[68000]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:15:27.511 INFO:teuthology.orchestra.run.smithi086.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T05:15:27.511 INFO:teuthology.orchestra.run.smithi086.stdout:alertmanager.a smithi086 *:9093,9094 running (3m) 56s ago 25m 28.1M - 0.25.0 c8568f914cd2 660ea62bb6b9 2024-04-02T05:15:27.511 INFO:teuthology.orchestra.run.smithi086.stdout:grafana.a smithi192 *:3000 running (3m) 72s ago 25m 79.2M - 9.4.12 2bacad6d85d8 259af9e63aa5 2024-04-02T05:15:27.511 INFO:teuthology.orchestra.run.smithi086.stdout:iscsi.foo.smithi086.rsbqwx smithi086 running (2m) 56s ago 24m 56.3M - 3.5 e1d6a67b021e fe728eb940f7 2024-04-02T05:15:27.511 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.x smithi192 *:8443,9283,8765 running (4m) 72s ago 29m 449M - 19.0.0-1979-g6cec793d 78f830190839 dd1e6d0f0a95 2024-04-02T05:15:27.511 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.y smithi086 *:8443,9283,8765 running (14m) 56s ago 31m 492M - 19.0.0-1979-g6cec793d 78f830190839 3dca54a018ab 2024-04-02T05:15:27.511 INFO:teuthology.orchestra.run.smithi086.stdout:mon.a smithi086 running (86s) 56s ago 31m 41.0M 2048M 19.0.0-1979-g6cec793d 78f830190839 8e478dd4da2d 2024-04-02T05:15:27.511 INFO:teuthology.orchestra.run.smithi086.stdout:mon.b smithi192 running (2m) 72s ago 30m 36.2M 2048M 19.0.0-1979-g6cec793d 78f830190839 4125e997188f 2024-04-02T05:15:27.512 INFO:teuthology.orchestra.run.smithi086.stdout:mon.c smithi086 running (62s) 56s ago 30m 22.3M 2048M 19.0.0-1979-g6cec793d 78f830190839 8180999b3142 2024-04-02T05:15:27.512 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.a smithi086 *:9100 running (4m) 56s ago 26m 15.5M - 1.5.0 0da6a335fe13 05a844569d7b 2024-04-02T05:15:27.512 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.b smithi192 *:9100 running (4m) 72s ago 26m 17.4M - 1.5.0 0da6a335fe13 1a7d6788d58b 2024-04-02T05:15:27.512 INFO:teuthology.orchestra.run.smithi086.stdout:osd.0 smithi086 running (29m) 56s ago 29m 58.1M 2992M 17.2.0 e1d6a67b021e 6d90c60d1c71 2024-04-02T05:15:27.512 INFO:teuthology.orchestra.run.smithi086.stdout:osd.1 smithi086 running (29m) 56s ago 29m 56.8M 2992M 17.2.0 e1d6a67b021e 203689f71603 2024-04-02T05:15:27.512 INFO:teuthology.orchestra.run.smithi086.stdout:osd.2 smithi086 running (28m) 56s ago 28m 50.7M 2992M 17.2.0 e1d6a67b021e 472c860a9af8 2024-04-02T05:15:27.512 INFO:teuthology.orchestra.run.smithi086.stdout:osd.3 smithi086 running (28m) 56s ago 28m 52.9M 2992M 17.2.0 e1d6a67b021e 327538dac44e 2024-04-02T05:15:27.512 INFO:teuthology.orchestra.run.smithi086.stdout:osd.4 smithi192 running (28m) 72s ago 28m 56.1M 3248M 17.2.0 e1d6a67b021e 6535d7919a2d 2024-04-02T05:15:27.512 INFO:teuthology.orchestra.run.smithi086.stdout:osd.5 smithi192 running (27m) 72s ago 27m 54.9M 3248M 17.2.0 e1d6a67b021e 7259249605f2 2024-04-02T05:15:27.512 INFO:teuthology.orchestra.run.smithi086.stdout:osd.6 smithi192 running (27m) 72s ago 27m 52.9M 3248M 17.2.0 e1d6a67b021e bb7d171b02a1 2024-04-02T05:15:27.512 INFO:teuthology.orchestra.run.smithi086.stdout:osd.7 smithi192 running (26m) 72s ago 26m 54.9M 3248M 17.2.0 e1d6a67b021e 95f5a0353999 2024-04-02T05:15:27.512 INFO:teuthology.orchestra.run.smithi086.stdout:prometheus.a smithi192 *:9095 running (3m) 72s ago 26m 65.1M - 2.43.0 a07b618ecd1d 8378fcdb60e3 2024-04-02T05:15:27.513 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi086.bxzxlp smithi086 *:8000 running (24m) 56s ago 24m 79.0M - 17.2.0 e1d6a67b021e 6725785fbc53 2024-04-02T05:15:27.513 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi192.pmlbot smithi192 *:8000 running (24m) 72s ago 24m 77.9M - 17.2.0 e1d6a67b021e 5ae3638f8b1c 2024-04-02T05:15:27.924 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:15:27.924 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": { 2024-04-02T05:15:27.925 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 3 2024-04-02T05:15:27.925 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:15:27.925 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": { 2024-04-02T05:15:27.925 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 2 2024-04-02T05:15:27.925 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:15:27.925 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": { 2024-04-02T05:15:27.925 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-02T05:15:27.925 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:15:27.925 INFO:teuthology.orchestra.run.smithi086.stdout: "rgw": { 2024-04-02T05:15:27.925 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-02T05:15:27.925 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:15:27.925 INFO:teuthology.orchestra.run.smithi086.stdout: "overall": { 2024-04-02T05:15:27.925 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-04-02T05:15:27.925 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 5 2024-04-02T05:15:27.926 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-04-02T05:15:27.926 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:15:28.123 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:27 smithi086 ceph-mon[88839]: from='client.54213 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:15:28.123 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:27 smithi086 ceph-mon[88839]: from='client.54219 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:15:28.123 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:28.123 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:15:28.124 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:28.124 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:28.124 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:28.124 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:28.124 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:28.124 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:28.124 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-02T05:15:28.124 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:27 smithi086 ceph-mon[86443]: from='client.54213 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:15:28.124 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:27 smithi086 ceph-mon[86443]: from='client.54219 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:15:28.124 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:28.125 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:15:28.125 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:28.125 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:28.125 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:28.125 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:28.125 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:28.125 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:28.125 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-02T05:15:28.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:27 smithi192 ceph-mon[68000]: from='client.54213 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:15:28.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:27 smithi192 ceph-mon[68000]: from='client.54219 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:15:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:15:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:28.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-02T05:15:28.275 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:15:28.275 INFO:teuthology.orchestra.run.smithi086.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", 2024-04-02T05:15:28.275 INFO:teuthology.orchestra.run.smithi086.stdout: "in_progress": true, 2024-04-02T05:15:28.275 INFO:teuthology.orchestra.run.smithi086.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (2 remaining).", 2024-04-02T05:15:28.275 INFO:teuthology.orchestra.run.smithi086.stdout: "services_complete": [], 2024-04-02T05:15:28.275 INFO:teuthology.orchestra.run.smithi086.stdout: "progress": "0/8 daemons upgraded", 2024-04-02T05:15:28.275 INFO:teuthology.orchestra.run.smithi086.stdout: "message": "Currently upgrading osd daemons", 2024-04-02T05:15:28.275 INFO:teuthology.orchestra.run.smithi086.stdout: "is_paused": false 2024-04-02T05:15:28.276 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:15:29.151 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:28 smithi086 ceph-mon[88839]: from='client.64168 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:15:29.151 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:28 smithi086 ceph-mon[88839]: Upgrade: Target is version 19.0.0-1979-g6cec793d (squid) 2024-04-02T05:15:29.151 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:28 smithi086 ceph-mon[88839]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7bd5b84926d4a0d6c51e5562e965bff211cec2740c9f4dec9b5e00a2bbb75eae'] 2024-04-02T05:15:29.151 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:28 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all mgr 2024-04-02T05:15:29.151 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:28 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all mon 2024-04-02T05:15:29.151 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:28 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all crash 2024-04-02T05:15:29.151 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:28 smithi086 ceph-mon[88839]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-02T05:15:29.151 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:28 smithi086 ceph-mon[88839]: Upgrade: osd.0 is safe to restart 2024-04-02T05:15:29.151 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:28 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/3398808101' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:29.151 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:28 smithi086 ceph-mon[88839]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:15:29.152 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:28 smithi086 ceph-mon[88839]: from='client.64180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:15:29.152 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:28 smithi086 ceph-mon[86443]: from='client.64168 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:15:29.152 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:28 smithi086 ceph-mon[86443]: Upgrade: Target is version 19.0.0-1979-g6cec793d (squid) 2024-04-02T05:15:29.152 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:28 smithi086 ceph-mon[86443]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7bd5b84926d4a0d6c51e5562e965bff211cec2740c9f4dec9b5e00a2bbb75eae'] 2024-04-02T05:15:29.152 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:28 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all mgr 2024-04-02T05:15:29.152 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:28 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all mon 2024-04-02T05:15:29.152 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:28 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all crash 2024-04-02T05:15:29.152 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:28 smithi086 ceph-mon[86443]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-02T05:15:29.152 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:28 smithi086 ceph-mon[86443]: Upgrade: osd.0 is safe to restart 2024-04-02T05:15:29.152 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:28 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/3398808101' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:29.152 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:28 smithi086 ceph-mon[86443]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:15:29.152 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:28 smithi086 ceph-mon[86443]: from='client.64180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:15:29.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:28 smithi192 ceph-mon[68000]: from='client.64168 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:15:29.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:28 smithi192 ceph-mon[68000]: Upgrade: Target is version 19.0.0-1979-g6cec793d (squid) 2024-04-02T05:15:29.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:28 smithi192 ceph-mon[68000]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7bd5b84926d4a0d6c51e5562e965bff211cec2740c9f4dec9b5e00a2bbb75eae'] 2024-04-02T05:15:29.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:28 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all mgr 2024-04-02T05:15:29.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:28 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all mon 2024-04-02T05:15:29.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:28 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all crash 2024-04-02T05:15:29.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:28 smithi192 ceph-mon[68000]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-02T05:15:29.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:28 smithi192 ceph-mon[68000]: Upgrade: osd.0 is safe to restart 2024-04-02T05:15:29.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:28 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/3398808101' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:29.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:28 smithi192 ceph-mon[68000]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:15:29.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:28 smithi192 ceph-mon[68000]: from='client.64180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:15:30.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:30 smithi086 ceph-mon[88839]: Upgrade: Updating osd.0 2024-04-02T05:15:30.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:30 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:30.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:30 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-02T05:15:30.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:30 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:15:30.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:30 smithi086 ceph-mon[88839]: Deploying daemon osd.0 on smithi086 2024-04-02T05:15:30.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:30 smithi086 ceph-mon[86443]: Upgrade: Updating osd.0 2024-04-02T05:15:30.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:30 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:30.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:30 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-02T05:15:30.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:30 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:15:30.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:30 smithi086 ceph-mon[86443]: Deploying daemon osd.0 on smithi086 2024-04-02T05:15:30.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:30 smithi192 ceph-mon[68000]: Upgrade: Updating osd.0 2024-04-02T05:15:30.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:30 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:30.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:30 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-02T05:15:30.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:30 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:15:30.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:30 smithi192 ceph-mon[68000]: Deploying daemon osd.0 on smithi086 2024-04-02T05:15:31.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:31 smithi086 ceph-mon[88839]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:15:31.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:31 smithi086 ceph-mon[86443]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:15:31.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:31 smithi192 ceph-mon[68000]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:15:32.833 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:32 smithi086 systemd[1]: Stopping Ceph osd.0 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:15:32.833 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:32 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0[32375]: 2024-04-02T05:15:32.718+0000 7efd72e5d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T05:15:32.833 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:32 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0[32375]: 2024-04-02T05:15:32.718+0000 7efd72e5d700 -1 osd.0 111 *** Got signal Terminated *** 2024-04-02T05:15:32.833 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:32 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0[32375]: 2024-04-02T05:15:32.718+0000 7efd72e5d700 -1 osd.0 111 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T05:15:33.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:15:33 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:15:33] "GET /metrics HTTP/1.1" 200 37948 "" "Prometheus/2.43.0" 2024-04-02T05:15:33.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:33 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:15:33.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:33 smithi086 ceph-mon[86443]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:15:33.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:33 smithi086 ceph-mon[86443]: osd.0 marked itself down and dead 2024-04-02T05:15:33.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:33 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:15:33.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:33 smithi086 ceph-mon[88839]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:15:33.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:33 smithi086 ceph-mon[88839]: osd.0 marked itself down and dead 2024-04-02T05:15:33.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:33 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:15:33.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:33 smithi192 ceph-mon[68000]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:15:33.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:33 smithi192 ceph-mon[68000]: osd.0 marked itself down and dead 2024-04-02T05:15:34.083 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:33 smithi086 podman[95462]: 2024-04-02 05:15:33.649047301 +0000 UTC m=+1.020072154 container died 6d90c60d1c71ae1da536a61e3b072abfda7c0b680c7c850b65600bd5b315927d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, architecture=x86_64, ceph=True, io.openshift.expose-services=, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, release=754, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, version=8, vendor=Red Hat, Inc.) 2024-04-02T05:15:34.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:34 smithi086 ceph-mon[88839]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-02T05:15:34.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:34 smithi086 ceph-mon[88839]: failed to encode map e112 with expected crc 2024-04-02T05:15:34.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:34 smithi086 ceph-mon[88839]: failed to encode map e112 with expected crc 2024-04-02T05:15:34.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:34 smithi086 ceph-mon[88839]: osdmap e112: 8 total, 7 up, 8 in 2024-04-02T05:15:34.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:34 smithi086 ceph-mon[88839]: failed to encode map e112 with expected crc 2024-04-02T05:15:34.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:34 smithi086 ceph-mon[88839]: failed to encode map e112 with expected crc 2024-04-02T05:15:34.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:34 smithi086 ceph-mon[86443]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-02T05:15:34.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:34 smithi086 ceph-mon[86443]: failed to encode map e112 with expected crc 2024-04-02T05:15:34.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:34 smithi086 ceph-mon[86443]: failed to encode map e112 with expected crc 2024-04-02T05:15:34.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:34 smithi086 ceph-mon[86443]: osdmap e112: 8 total, 7 up, 8 in 2024-04-02T05:15:34.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:34 smithi086 ceph-mon[86443]: failed to encode map e112 with expected crc 2024-04-02T05:15:34.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:34 smithi086 ceph-mon[86443]: failed to encode map e112 with expected crc 2024-04-02T05:15:34.584 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:34 smithi086 podman[95462]: 2024-04-02 05:15:34.095165308 +0000 UTC m=+1.466190170 container cleanup 6d90c60d1c71ae1da536a61e3b072abfda7c0b680c7c850b65600bd5b315927d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, architecture=x86_64, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, distribution-scope=public, release=754, RELEASE=HEAD, name=centos-stream, vcs-type=git, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-02T05:15:34.584 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:34 smithi086 bash[95462]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0 2024-04-02T05:15:34.584 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:34 smithi086 podman[95479]: 2024-04-02 05:15:34.254491385 +0000 UTC m=+0.603361878 container remove 6d90c60d1c71ae1da536a61e3b072abfda7c0b680c7c850b65600bd5b315927d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0, version=8, architecture=x86_64, GIT_BRANCH=HEAD, ceph=True, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, name=centos-stream, com.redhat.component=centos-stream-container, distribution-scope=public, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8) 2024-04-02T05:15:34.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:34 smithi192 ceph-mon[68000]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-02T05:15:34.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:34 smithi192 ceph-mon[68000]: failed to encode map e112 with expected crc 2024-04-02T05:15:34.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:34 smithi192 ceph-mon[68000]: failed to encode map e112 with expected crc 2024-04-02T05:15:34.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:34 smithi192 ceph-mon[68000]: osdmap e112: 8 total, 7 up, 8 in 2024-04-02T05:15:34.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:34 smithi192 ceph-mon[68000]: failed to encode map e112 with expected crc 2024-04-02T05:15:34.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:34 smithi192 ceph-mon[68000]: failed to encode map e112 with expected crc 2024-04-02T05:15:35.010 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:34 smithi086 podman[95571]: 2024-04-02 05:15:34.569206232 +0000 UTC m=+0.017884811 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:15:35.010 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:34 smithi086 podman[95571]: 2024-04-02 05:15:34.689061554 +0000 UTC m=+0.137740118 container create ccfd0c9f2f92977a95eb4f99e0470c8de80314b3405531130532d0d7ef49d631 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-6cec793, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-02T05:15:35.010 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:34 smithi086 podman[95571]: 2024-04-02 05:15:34.908128779 +0000 UTC m=+0.356807343 container init ccfd0c9f2f92977a95eb4f99e0470c8de80314b3405531130532d0d7ef49d631 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0-deactivate, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-6cec793, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-02T05:15:35.011 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:34 smithi086 podman[95571]: 2024-04-02 05:15:34.911634012 +0000 UTC m=+0.360312586 container start ccfd0c9f2f92977a95eb4f99e0470c8de80314b3405531130532d0d7ef49d631 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=squid-6cec793, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-02T05:15:35.011 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:35 smithi086 podman[95571]: 2024-04-02 05:15:35.008085008 +0000 UTC m=+0.456763575 container attach ccfd0c9f2f92977a95eb4f99e0470c8de80314b3405531130532d0d7ef49d631 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-6cec793, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-02T05:15:35.330 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:35 smithi086 podman[95571]: 2024-04-02 05:15:35.201450702 +0000 UTC m=+0.650129267 container died ccfd0c9f2f92977a95eb4f99e0470c8de80314b3405531130532d0d7ef49d631 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0-deactivate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-6cec793) 2024-04-02T05:15:35.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:35 smithi086 ceph-mon[86443]: pgmap v46: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:15:35.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:35 smithi086 ceph-mon[86443]: failed to encode map e113 with expected crc 2024-04-02T05:15:35.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:35 smithi086 ceph-mon[86443]: failed to encode map e113 with expected crc 2024-04-02T05:15:35.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:35 smithi086 ceph-mon[86443]: osdmap e113: 8 total, 7 up, 8 in 2024-04-02T05:15:35.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:35 smithi086 ceph-mon[86443]: failed to encode map e113 with expected crc 2024-04-02T05:15:35.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:35 smithi086 ceph-mon[86443]: failed to encode map e113 with expected crc 2024-04-02T05:15:35.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:35 smithi086 ceph-mon[86443]: failed to encode map e113 with expected crc 2024-04-02T05:15:35.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:35 smithi086 ceph-mon[86443]: failed to encode map e113 with expected crc 2024-04-02T05:15:35.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:35 smithi086 ceph-mon[86443]: failed to encode map e113 with expected crc 2024-04-02T05:15:35.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:35 smithi086 ceph-mon[86443]: failed to encode map e113 with expected crc 2024-04-02T05:15:35.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:35 smithi086 ceph-mon[88839]: pgmap v46: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:15:35.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:35 smithi086 ceph-mon[88839]: failed to encode map e113 with expected crc 2024-04-02T05:15:35.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:35 smithi086 ceph-mon[88839]: failed to encode map e113 with expected crc 2024-04-02T05:15:35.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:35 smithi086 ceph-mon[88839]: osdmap e113: 8 total, 7 up, 8 in 2024-04-02T05:15:35.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:35 smithi086 ceph-mon[88839]: failed to encode map e113 with expected crc 2024-04-02T05:15:35.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:35 smithi086 ceph-mon[88839]: failed to encode map e113 with expected crc 2024-04-02T05:15:35.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:35 smithi086 ceph-mon[88839]: failed to encode map e113 with expected crc 2024-04-02T05:15:35.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:35 smithi086 ceph-mon[88839]: failed to encode map e113 with expected crc 2024-04-02T05:15:35.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:35 smithi086 ceph-mon[88839]: failed to encode map e113 with expected crc 2024-04-02T05:15:35.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:35 smithi086 ceph-mon[88839]: failed to encode map e113 with expected crc 2024-04-02T05:15:35.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:35 smithi192 ceph-mon[68000]: pgmap v46: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:15:35.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:35 smithi192 ceph-mon[68000]: failed to encode map e113 with expected crc 2024-04-02T05:15:35.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:35 smithi192 ceph-mon[68000]: failed to encode map e113 with expected crc 2024-04-02T05:15:35.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:35 smithi192 ceph-mon[68000]: osdmap e113: 8 total, 7 up, 8 in 2024-04-02T05:15:35.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:35 smithi192 ceph-mon[68000]: failed to encode map e113 with expected crc 2024-04-02T05:15:35.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:35 smithi192 ceph-mon[68000]: failed to encode map e113 with expected crc 2024-04-02T05:15:35.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:35 smithi192 ceph-mon[68000]: failed to encode map e113 with expected crc 2024-04-02T05:15:35.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:35 smithi192 ceph-mon[68000]: failed to encode map e113 with expected crc 2024-04-02T05:15:35.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:35 smithi192 ceph-mon[68000]: failed to encode map e113 with expected crc 2024-04-02T05:15:35.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:35 smithi192 ceph-mon[68000]: failed to encode map e113 with expected crc 2024-04-02T05:15:36.299 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:35 smithi086 podman[95598]: 2024-04-02 05:15:35.837042251 +0000 UTC m=+0.627210241 container remove ccfd0c9f2f92977a95eb4f99e0470c8de80314b3405531130532d0d7ef49d631 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-6cec793, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-04-02T05:15:36.299 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:35 smithi086 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.0.service: Deactivated successfully. 2024-04-02T05:15:36.299 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:35 smithi086 systemd[1]: Stopped Ceph osd.0 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:15:36.299 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:35 smithi086 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.0.service: Consumed 13.856s CPU time. 2024-04-02T05:15:36.299 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:36 smithi086 systemd[1]: Starting Ceph osd.0 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:15:36.583 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:36 smithi086 podman[95704]: 2024-04-02 05:15:36.200172034 +0000 UTC m=+0.017837115 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:15:36.583 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:36 smithi086 podman[95704]: 2024-04-02 05:15:36.32974301 +0000 UTC m=+0.147408067 container create 7914b133ed6b7d77b6f567f3e92fea2a857dedea7b60eea1fb584fab7a484e6c (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0-activate, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=squid-6cec793, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-04-02T05:15:36.583 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:36 smithi086 podman[95704]: 2024-04-02 05:15:36.532488994 +0000 UTC m=+0.350154052 container init 7914b133ed6b7d77b6f567f3e92fea2a857dedea7b60eea1fb584fab7a484e6c (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0-activate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-6cec793, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-04-02T05:15:36.583 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:36 smithi086 podman[95704]: 2024-04-02 05:15:36.535563523 +0000 UTC m=+0.353228589 container start 7914b133ed6b7d77b6f567f3e92fea2a857dedea7b60eea1fb584fab7a484e6c (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0-activate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=squid-6cec793, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=) 2024-04-02T05:15:36.981 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:36 smithi086 podman[95704]: 2024-04-02 05:15:36.615967248 +0000 UTC m=+0.433632317 container attach 7914b133ed6b7d77b6f567f3e92fea2a857dedea7b60eea1fb584fab7a484e6c (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0-activate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=squid-6cec793, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD) 2024-04-02T05:15:37.296 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:36 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0-activate[95719]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-02T05:15:37.296 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:36 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0-activate[95719]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-02T05:15:37.296 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:36 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0-activate[95719]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-02T05:15:37.297 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:36 smithi086 bash[95704]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-02T05:15:37.297 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:36 smithi086 bash[95704]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-02T05:15:37.297 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:36 smithi086 bash[95704]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-02T05:15:37.297 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:36 smithi086 bash[95704]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-02T05:15:37.297 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:36 smithi086 bash[95704]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-02T05:15:37.297 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:36 smithi086 bash[95704]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-02T05:15:37.297 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:36 smithi086 bash[95704]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-02T05:15:37.297 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:36 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0-activate[95719]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-02T05:15:37.297 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:36 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0-activate[95719]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-02T05:15:37.297 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:36 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0-activate[95719]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-02T05:15:37.297 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:36 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0-activate[95719]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-02T05:15:37.297 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:37 smithi086 podman[95704]: 2024-04-02 05:15:37.00287391 +0000 UTC m=+0.820538969 container died 7914b133ed6b7d77b6f567f3e92fea2a857dedea7b60eea1fb584fab7a484e6c (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0-activate, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , RELEASE=squid-6cec793, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-04-02T05:15:37.580 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:37 smithi086 ceph-mon[88839]: pgmap v48: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-02T05:15:37.580 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:37 smithi086 ceph-mon[86443]: pgmap v48: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-02T05:15:37.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:37 smithi192 ceph-mon[68000]: pgmap v48: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-02T05:15:37.833 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:37 smithi086 podman[95855]: 2024-04-02 05:15:37.57808786 +0000 UTC m=+0.567445151 container remove 7914b133ed6b7d77b6f567f3e92fea2a857dedea7b60eea1fb584fab7a484e6c (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0-activate, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-6cec793, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-02T05:15:37.833 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:37 smithi086 podman[95917]: 2024-04-02 05:15:37.787234567 +0000 UTC m=+0.112875041 container create e521a0ef7dafe5fe9b0fb6ca00f9aadb83ae8f63747c3197f29c0539c46f10a5 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0, RELEASE=squid-6cec793, org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True) 2024-04-02T05:15:37.833 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:37 smithi086 podman[95917]: 2024-04-02 05:15:37.692420187 +0000 UTC m=+0.018060652 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:15:38.333 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:37 smithi086 podman[95917]: 2024-04-02 05:15:37.997910461 +0000 UTC m=+0.323550895 container init e521a0ef7dafe5fe9b0fb6ca00f9aadb83ae8f63747c3197f29c0539c46f10a5 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , RELEASE=squid-6cec793, GIT_BRANCH=HEAD) 2024-04-02T05:15:38.333 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:38 smithi086 podman[95917]: 2024-04-02 05:15:38.000970154 +0000 UTC m=+0.326610596 container start e521a0ef7dafe5fe9b0fb6ca00f9aadb83ae8f63747c3197f29c0539c46f10a5 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=squid-6cec793, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-04-02T05:15:38.333 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:38 smithi086 ceph-osd[95937]: -- 172.21.15.86:0/2368922232 <== mon.0 v2:172.21.15.86:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55680c1b1680 con 0x55680c17ac00 2024-04-02T05:15:38.333 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:38 smithi086 bash[95917]: e521a0ef7dafe5fe9b0fb6ca00f9aadb83ae8f63747c3197f29c0539c46f10a5 2024-04-02T05:15:38.333 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:38 smithi086 systemd[1]: Started Ceph osd.0 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:15:38.833 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:38 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0[95933]: 2024-04-02T05:15:38.549+0000 7fcbaa8b17c0 -1 Falling back to public interface 2024-04-02T05:15:39.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:39 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:39.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:39 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:39.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:39 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:39.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:39 smithi086 ceph-mon[86443]: pgmap v49: 161 pgs: 23 active+undersized+degraded, 35 active+undersized, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 77/723 objects degraded (10.650%) 2024-04-02T05:15:39.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:39 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:39.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:39 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:39.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:39 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:39.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:39 smithi086 ceph-mon[88839]: pgmap v49: 161 pgs: 23 active+undersized+degraded, 35 active+undersized, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 77/723 objects degraded (10.650%) 2024-04-02T05:15:39.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:39 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:39.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:39 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:39.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:39 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:39.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:39 smithi192 ceph-mon[68000]: pgmap v49: 161 pgs: 23 active+undersized+degraded, 35 active+undersized, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 77/723 objects degraded (10.650%) 2024-04-02T05:15:40.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:40 smithi192 ceph-mon[68000]: Health check failed: Degraded data redundancy: 77/723 objects degraded (10.650%), 23 pgs degraded (PG_DEGRADED) 2024-04-02T05:15:40.562 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:40 smithi086 ceph-mon[88839]: Health check failed: Degraded data redundancy: 77/723 objects degraded (10.650%), 23 pgs degraded (PG_DEGRADED) 2024-04-02T05:15:40.562 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:40 smithi086 ceph-mon[86443]: Health check failed: Degraded data redundancy: 77/723 objects degraded (10.650%), 23 pgs degraded (PG_DEGRADED) 2024-04-02T05:15:41.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:41 smithi086 ceph-mon[88839]: pgmap v50: 161 pgs: 23 active+undersized+degraded, 35 active+undersized, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 77/723 objects degraded (10.650%) 2024-04-02T05:15:41.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:41 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:41.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:41 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:15:41.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:41 smithi086 ceph-mon[88839]: from='osd.0 [v2:172.21.15.86:6802/3837258523,v1:172.21.15.86:6803/3837258523]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-02T05:15:41.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:41 smithi086 ceph-mon[88839]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-02T05:15:41.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:41 smithi086 ceph-mon[86443]: pgmap v50: 161 pgs: 23 active+undersized+degraded, 35 active+undersized, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 77/723 objects degraded (10.650%) 2024-04-02T05:15:41.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:41 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:41.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:41 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:15:41.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:41 smithi086 ceph-mon[86443]: from='osd.0 [v2:172.21.15.86:6802/3837258523,v1:172.21.15.86:6803/3837258523]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-02T05:15:41.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:41 smithi086 ceph-mon[86443]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-02T05:15:41.584 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:41 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0[95933]: 2024-04-02T05:15:41.236+0000 7fcbaa8b17c0 -1 osd.0 0 read_superblock omap replica is missing. 2024-04-02T05:15:41.584 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:41 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0[95933]: 2024-04-02T05:15:41.268+0000 7fcbaa8b17c0 -1 osd.0 111 log_to_monitors true 2024-04-02T05:15:41.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:41 smithi192 ceph-mon[68000]: pgmap v50: 161 pgs: 23 active+undersized+degraded, 35 active+undersized, 103 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 77/723 objects degraded (10.650%) 2024-04-02T05:15:41.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:41 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:41.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:41 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:15:41.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:41 smithi192 ceph-mon[68000]: from='osd.0 [v2:172.21.15.86:6802/3837258523,v1:172.21.15.86:6803/3837258523]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-02T05:15:41.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:41 smithi192 ceph-mon[68000]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-02T05:15:42.308 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:15:42 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0[95933]: 2024-04-02T05:15:42.171+0000 7fcb9bd09700 -1 osd.0 111 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T05:15:43.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:43 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:15:43.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:43 smithi192 ceph-mon[68000]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-02T05:15:43.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:43 smithi192 ceph-mon[68000]: failed to encode map e114 with expected crc 2024-04-02T05:15:43.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:43 smithi192 ceph-mon[68000]: failed to encode map e114 with expected crc 2024-04-02T05:15:43.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:43 smithi192 ceph-mon[68000]: osdmap e114: 8 total, 7 up, 8 in 2024-04-02T05:15:43.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:43 smithi192 ceph-mon[68000]: from='osd.0 [v2:172.21.15.86:6802/3837258523,v1:172.21.15.86:6803/3837258523]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T05:15:43.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:43 smithi192 ceph-mon[68000]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T05:15:43.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:43 smithi192 ceph-mon[68000]: pgmap v52: 161 pgs: 23 active+undersized+degraded, 35 active+undersized, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 77/723 objects degraded (10.650%) 2024-04-02T05:15:43.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:43 smithi192 ceph-mon[68000]: failed to encode map e114 with expected crc 2024-04-02T05:15:43.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:43 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:43.429 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:43 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:43.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:15:43 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:15:43] "GET /metrics HTTP/1.1" 200 38147 "" "Prometheus/2.43.0" 2024-04-02T05:15:43.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:43 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:15:43.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:43 smithi086 ceph-mon[86443]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-02T05:15:43.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:43 smithi086 ceph-mon[86443]: failed to encode map e114 with expected crc 2024-04-02T05:15:43.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:43 smithi086 ceph-mon[86443]: failed to encode map e114 with expected crc 2024-04-02T05:15:43.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:43 smithi086 ceph-mon[86443]: osdmap e114: 8 total, 7 up, 8 in 2024-04-02T05:15:43.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:43 smithi086 ceph-mon[86443]: from='osd.0 [v2:172.21.15.86:6802/3837258523,v1:172.21.15.86:6803/3837258523]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T05:15:43.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:43 smithi086 ceph-mon[86443]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T05:15:43.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:43 smithi086 ceph-mon[86443]: pgmap v52: 161 pgs: 23 active+undersized+degraded, 35 active+undersized, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 77/723 objects degraded (10.650%) 2024-04-02T05:15:43.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:43 smithi086 ceph-mon[86443]: failed to encode map e114 with expected crc 2024-04-02T05:15:43.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:43 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:43.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:43 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:43.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:43 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:15:43.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:43 smithi086 ceph-mon[88839]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-02T05:15:43.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:43 smithi086 ceph-mon[88839]: failed to encode map e114 with expected crc 2024-04-02T05:15:43.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:43 smithi086 ceph-mon[88839]: failed to encode map e114 with expected crc 2024-04-02T05:15:43.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:43 smithi086 ceph-mon[88839]: osdmap e114: 8 total, 7 up, 8 in 2024-04-02T05:15:43.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:43 smithi086 ceph-mon[88839]: from='osd.0 [v2:172.21.15.86:6802/3837258523,v1:172.21.15.86:6803/3837258523]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T05:15:43.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:43 smithi086 ceph-mon[88839]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T05:15:43.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:43 smithi086 ceph-mon[88839]: pgmap v52: 161 pgs: 23 active+undersized+degraded, 35 active+undersized, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 77/723 objects degraded (10.650%) 2024-04-02T05:15:43.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:43 smithi086 ceph-mon[88839]: failed to encode map e114 with expected crc 2024-04-02T05:15:43.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:43 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:43.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:43 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:44.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:44 smithi192 ceph-mon[68000]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-02T05:15:44.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:44 smithi192 ceph-mon[68000]: failed to encode map e115 with expected crc 2024-04-02T05:15:44.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:44 smithi192 ceph-mon[68000]: failed to encode map e115 with expected crc 2024-04-02T05:15:44.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:44 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T05:15:44.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:44 smithi192 ceph-mon[68000]: osd.0 [v2:172.21.15.86:6802/3837258523,v1:172.21.15.86:6803/3837258523] boot 2024-04-02T05:15:44.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:44 smithi192 ceph-mon[68000]: osdmap e115: 8 total, 8 up, 8 in 2024-04-02T05:15:44.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:44 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:44.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:44 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:44.471 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:44 smithi086 ceph-mon[86443]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-02T05:15:44.471 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:44 smithi086 ceph-mon[86443]: failed to encode map e115 with expected crc 2024-04-02T05:15:44.471 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:44 smithi086 ceph-mon[86443]: failed to encode map e115 with expected crc 2024-04-02T05:15:44.471 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:44 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T05:15:44.471 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:44 smithi086 ceph-mon[86443]: osd.0 [v2:172.21.15.86:6802/3837258523,v1:172.21.15.86:6803/3837258523] boot 2024-04-02T05:15:44.471 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:44 smithi086 ceph-mon[86443]: osdmap e115: 8 total, 8 up, 8 in 2024-04-02T05:15:44.471 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:44 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:44.471 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:44 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:44.472 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:44 smithi086 ceph-mon[88839]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-02T05:15:44.472 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:44 smithi086 ceph-mon[88839]: failed to encode map e115 with expected crc 2024-04-02T05:15:44.472 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:44 smithi086 ceph-mon[88839]: failed to encode map e115 with expected crc 2024-04-02T05:15:44.472 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:44 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T05:15:44.472 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:44 smithi086 ceph-mon[88839]: osd.0 [v2:172.21.15.86:6802/3837258523,v1:172.21.15.86:6803/3837258523] boot 2024-04-02T05:15:44.472 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:44 smithi086 ceph-mon[88839]: osdmap e115: 8 total, 8 up, 8 in 2024-04-02T05:15:44.472 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:44 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:44.472 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:44 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:45.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:45 smithi192 ceph-mon[68000]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:45 smithi192 ceph-mon[68000]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:45 smithi192 ceph-mon[68000]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:45 smithi192 ceph-mon[68000]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:45 smithi192 ceph-mon[68000]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:45 smithi192 ceph-mon[68000]: osdmap e116: 8 total, 8 up, 8 in 2024-04-02T05:15:45.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:45 smithi192 ceph-mon[68000]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:45 smithi192 ceph-mon[68000]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:45 smithi192 ceph-mon[68000]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:45 smithi192 ceph-mon[68000]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:45 smithi192 ceph-mon[68000]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:45 smithi192 ceph-mon[68000]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:45 smithi192 ceph-mon[68000]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:45 smithi192 ceph-mon[68000]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:45 smithi192 ceph-mon[68000]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:45 smithi192 ceph-mon[68000]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:45 smithi192 ceph-mon[68000]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:45 smithi192 ceph-mon[68000]: pgmap v55: 161 pgs: 23 active+undersized+degraded, 35 active+undersized, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 77/723 objects degraded (10.650%) 2024-04-02T05:15:45.441 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:45 smithi086 ceph-mon[86443]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.441 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:45 smithi086 ceph-mon[86443]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.441 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:45 smithi086 ceph-mon[86443]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.441 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:45 smithi086 ceph-mon[86443]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.441 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:45 smithi086 ceph-mon[86443]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.441 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:45 smithi086 ceph-mon[86443]: osdmap e116: 8 total, 8 up, 8 in 2024-04-02T05:15:45.441 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:45 smithi086 ceph-mon[86443]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.442 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:45 smithi086 ceph-mon[86443]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.442 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:45 smithi086 ceph-mon[86443]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.442 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:45 smithi086 ceph-mon[86443]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.442 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:45 smithi086 ceph-mon[86443]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.442 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:45 smithi086 ceph-mon[86443]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.442 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:45 smithi086 ceph-mon[86443]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.442 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:45 smithi086 ceph-mon[86443]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.442 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:45 smithi086 ceph-mon[86443]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.442 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:45 smithi086 ceph-mon[86443]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.442 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:45 smithi086 ceph-mon[86443]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.442 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:45 smithi086 ceph-mon[86443]: pgmap v55: 161 pgs: 23 active+undersized+degraded, 35 active+undersized, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 77/723 objects degraded (10.650%) 2024-04-02T05:15:45.442 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:45 smithi086 ceph-mon[88839]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.443 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:45 smithi086 ceph-mon[88839]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.443 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:45 smithi086 ceph-mon[88839]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.443 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:45 smithi086 ceph-mon[88839]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.443 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:45 smithi086 ceph-mon[88839]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.443 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:45 smithi086 ceph-mon[88839]: osdmap e116: 8 total, 8 up, 8 in 2024-04-02T05:15:45.443 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:45 smithi086 ceph-mon[88839]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.443 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:45 smithi086 ceph-mon[88839]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.443 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:45 smithi086 ceph-mon[88839]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.443 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:45 smithi086 ceph-mon[88839]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.443 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:45 smithi086 ceph-mon[88839]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.443 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:45 smithi086 ceph-mon[88839]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.443 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:45 smithi086 ceph-mon[88839]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.443 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:45 smithi086 ceph-mon[88839]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.443 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:45 smithi086 ceph-mon[88839]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.443 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:45 smithi086 ceph-mon[88839]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.444 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:45 smithi086 ceph-mon[88839]: failed to encode map e116 with expected crc 2024-04-02T05:15:45.444 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:45 smithi086 ceph-mon[88839]: pgmap v55: 161 pgs: 23 active+undersized+degraded, 35 active+undersized, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 77/723 objects degraded (10.650%) 2024-04-02T05:15:47.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:15:47 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:15:47.293+0000 7fe19d012700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-04-02T05:15:47.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:47 smithi086 ceph-mon[88839]: pgmap v56: 161 pgs: 23 active+undersized+degraded, 35 active+undersized, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 77/723 objects degraded (10.650%) 2024-04-02T05:15:47.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:47 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:47.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:47 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:47.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:47 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:15:47.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:47 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:15:47.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:47 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:47.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:47 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:15:47.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:47 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:47.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:47 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:47.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:47 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:47.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:47 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-02T05:15:47.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:47 smithi086 ceph-mon[86443]: pgmap v56: 161 pgs: 23 active+undersized+degraded, 35 active+undersized, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 77/723 objects degraded (10.650%) 2024-04-02T05:15:47.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:47 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:47.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:47 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:47.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:47 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:15:47.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:47 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:15:47.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:47 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:47.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:47 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:15:47.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:47 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:47.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:47 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:47.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:47 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:47.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:47 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-02T05:15:47.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:47 smithi192 ceph-mon[68000]: pgmap v56: 161 pgs: 23 active+undersized+degraded, 35 active+undersized, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 77/723 objects degraded (10.650%) 2024-04-02T05:15:47.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:47 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:47.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:47 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:47.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:47 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:15:47.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:47 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:15:47.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:47 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:47.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:47 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:15:47.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:47 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:47.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:47 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:47.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:47 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:15:47.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:47 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-02T05:15:48.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:48 smithi086 ceph-mon[88839]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-02T05:15:48.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:48 smithi086 ceph-mon[88839]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-04-02T05:15:48.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:48 smithi086 ceph-mon[86443]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-02T05:15:48.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:48 smithi086 ceph-mon[86443]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-04-02T05:15:48.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:48 smithi192 ceph-mon[68000]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-02T05:15:48.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:48 smithi192 ceph-mon[68000]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-04-02T05:15:49.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:49 smithi086 ceph-mon[88839]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 836 B/s rd, 0 op/s 2024-04-02T05:15:49.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:49 smithi086 ceph-mon[88839]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 77/723 objects degraded (10.650%), 23 pgs degraded) 2024-04-02T05:15:49.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:49 smithi086 ceph-mon[88839]: Cluster is now healthy 2024-04-02T05:15:49.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:49 smithi086 ceph-mon[86443]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 836 B/s rd, 0 op/s 2024-04-02T05:15:49.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:49 smithi086 ceph-mon[86443]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 77/723 objects degraded (10.650%), 23 pgs degraded) 2024-04-02T05:15:49.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:49 smithi086 ceph-mon[86443]: Cluster is now healthy 2024-04-02T05:15:49.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:49 smithi192 ceph-mon[68000]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 836 B/s rd, 0 op/s 2024-04-02T05:15:49.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:49 smithi192 ceph-mon[68000]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 77/723 objects degraded (10.650%), 23 pgs degraded) 2024-04-02T05:15:49.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:49 smithi192 ceph-mon[68000]: Cluster is now healthy 2024-04-02T05:15:51.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:51 smithi086 ceph-mon[88839]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-02T05:15:51.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:51 smithi086 ceph-mon[86443]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-02T05:15:51.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:51 smithi192 ceph-mon[68000]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-02T05:15:53.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:53 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:15:53.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:53 smithi086 ceph-mon[86443]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:15:53.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:53 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:15:53.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:53 smithi086 ceph-mon[88839]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:15:53.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:15:53 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:15:53] "GET /metrics HTTP/1.1" 200 38147 "" "Prometheus/2.43.0" 2024-04-02T05:15:53.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:53 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:15:53.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:53 smithi192 ceph-mon[68000]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T05:15:55.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:55 smithi192 ceph-mon[68000]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1012 B/s rd, 0 op/s 2024-04-02T05:15:55.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:55 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:55.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:55 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:15:55.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:55 smithi086 ceph-mon[86443]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1012 B/s rd, 0 op/s 2024-04-02T05:15:55.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:55 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:55.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:55 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:15:55.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:55 smithi086 ceph-mon[88839]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1012 B/s rd, 0 op/s 2024-04-02T05:15:55.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:55 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:55.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:55 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:15:56.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:56 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:56.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:56 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:56.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:56 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:15:57.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:57 smithi192 ceph-mon[68000]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:15:57.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:57 smithi086 ceph-mon[86443]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:15:57.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:57 smithi086 ceph-mon[88839]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:15:58.626 INFO:teuthology.orchestra.run.smithi086.stdout:true 2024-04-02T05:15:59.273 INFO:teuthology.orchestra.run.smithi086.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T05:15:59.273 INFO:teuthology.orchestra.run.smithi086.stdout:alertmanager.a smithi086 *:9093,9094 running (4m) 16s ago 25m 28.1M - 0.25.0 c8568f914cd2 660ea62bb6b9 2024-04-02T05:15:59.274 INFO:teuthology.orchestra.run.smithi086.stdout:grafana.a smithi192 *:3000 running (3m) 104s ago 25m 79.2M - 9.4.12 2bacad6d85d8 259af9e63aa5 2024-04-02T05:15:59.274 INFO:teuthology.orchestra.run.smithi086.stdout:iscsi.foo.smithi086.rsbqwx smithi086 running (3m) 16s ago 25m 56.6M - 3.5 e1d6a67b021e fe728eb940f7 2024-04-02T05:15:59.274 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.x smithi192 *:8443,9283,8765 running (5m) 104s ago 30m 449M - 19.0.0-1979-g6cec793d 78f830190839 dd1e6d0f0a95 2024-04-02T05:15:59.274 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.y smithi086 *:8443,9283,8765 running (14m) 16s ago 32m 499M - 19.0.0-1979-g6cec793d 78f830190839 3dca54a018ab 2024-04-02T05:15:59.274 INFO:teuthology.orchestra.run.smithi086.stdout:mon.a smithi086 running (117s) 16s ago 32m 45.0M 2048M 19.0.0-1979-g6cec793d 78f830190839 8e478dd4da2d 2024-04-02T05:15:59.275 INFO:teuthology.orchestra.run.smithi086.stdout:mon.b smithi192 running (2m) 104s ago 30m 36.2M 2048M 19.0.0-1979-g6cec793d 78f830190839 4125e997188f 2024-04-02T05:15:59.275 INFO:teuthology.orchestra.run.smithi086.stdout:mon.c smithi086 running (94s) 16s ago 30m 36.0M 2048M 19.0.0-1979-g6cec793d 78f830190839 8180999b3142 2024-04-02T05:15:59.275 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.a smithi086 *:9100 running (4m) 16s ago 26m 15.7M - 1.5.0 0da6a335fe13 05a844569d7b 2024-04-02T05:15:59.275 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.b smithi192 *:9100 running (4m) 104s ago 26m 17.4M - 1.5.0 0da6a335fe13 1a7d6788d58b 2024-04-02T05:15:59.275 INFO:teuthology.orchestra.run.smithi086.stdout:osd.0 smithi086 running (21s) 16s ago 30m 12.7M 2992M 19.0.0-1979-g6cec793d 78f830190839 e521a0ef7daf 2024-04-02T05:15:59.275 INFO:teuthology.orchestra.run.smithi086.stdout:osd.1 smithi086 running (29m) 16s ago 29m 57.6M 2992M 17.2.0 e1d6a67b021e 203689f71603 2024-04-02T05:15:59.275 INFO:teuthology.orchestra.run.smithi086.stdout:osd.2 smithi086 running (29m) 16s ago 29m 51.6M 2992M 17.2.0 e1d6a67b021e 472c860a9af8 2024-04-02T05:15:59.276 INFO:teuthology.orchestra.run.smithi086.stdout:osd.3 smithi086 running (28m) 16s ago 28m 53.8M 2992M 17.2.0 e1d6a67b021e 327538dac44e 2024-04-02T05:15:59.276 INFO:teuthology.orchestra.run.smithi086.stdout:osd.4 smithi192 running (28m) 104s ago 28m 56.1M 3248M 17.2.0 e1d6a67b021e 6535d7919a2d 2024-04-02T05:15:59.276 INFO:teuthology.orchestra.run.smithi086.stdout:osd.5 smithi192 running (28m) 104s ago 28m 54.9M 3248M 17.2.0 e1d6a67b021e 7259249605f2 2024-04-02T05:15:59.276 INFO:teuthology.orchestra.run.smithi086.stdout:osd.6 smithi192 running (27m) 104s ago 27m 52.9M 3248M 17.2.0 e1d6a67b021e bb7d171b02a1 2024-04-02T05:15:59.276 INFO:teuthology.orchestra.run.smithi086.stdout:osd.7 smithi192 running (27m) 104s ago 27m 54.9M 3248M 17.2.0 e1d6a67b021e 95f5a0353999 2024-04-02T05:15:59.276 INFO:teuthology.orchestra.run.smithi086.stdout:prometheus.a smithi192 *:9095 running (4m) 104s ago 26m 65.1M - 2.43.0 a07b618ecd1d 8378fcdb60e3 2024-04-02T05:15:59.276 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi086.bxzxlp smithi086 *:8000 running (25m) 16s ago 25m 79.2M - 17.2.0 e1d6a67b021e 6725785fbc53 2024-04-02T05:15:59.276 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi192.pmlbot smithi192 *:8000 running (25m) 104s ago 25m 77.9M - 17.2.0 e1d6a67b021e 5ae3638f8b1c 2024-04-02T05:15:59.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:15:59 smithi086 ceph-mon[86443]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:15:59.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:15:59 smithi086 ceph-mon[88839]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:15:59.665 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:15:59.665 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": { 2024-04-02T05:15:59.665 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 3 2024-04-02T05:15:59.665 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:15:59.665 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": { 2024-04-02T05:15:59.665 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 2 2024-04-02T05:15:59.665 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:15:59.665 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": { 2024-04-02T05:15:59.665 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-04-02T05:15:59.666 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 1 2024-04-02T05:15:59.666 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:15:59.666 INFO:teuthology.orchestra.run.smithi086.stdout: "rgw": { 2024-04-02T05:15:59.666 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-02T05:15:59.666 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:15:59.666 INFO:teuthology.orchestra.run.smithi086.stdout: "overall": { 2024-04-02T05:15:59.666 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-04-02T05:15:59.666 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 6 2024-04-02T05:15:59.666 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-04-02T05:15:59.666 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:15:59.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:15:59 smithi192 ceph-mon[68000]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:15:59.994 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:15:59.994 INFO:teuthology.orchestra.run.smithi086.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", 2024-04-02T05:15:59.994 INFO:teuthology.orchestra.run.smithi086.stdout: "in_progress": true, 2024-04-02T05:15:59.994 INFO:teuthology.orchestra.run.smithi086.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (1 remaining).", 2024-04-02T05:15:59.994 INFO:teuthology.orchestra.run.smithi086.stdout: "services_complete": [], 2024-04-02T05:15:59.994 INFO:teuthology.orchestra.run.smithi086.stdout: "progress": "1/8 daemons upgraded", 2024-04-02T05:15:59.994 INFO:teuthology.orchestra.run.smithi086.stdout: "message": "Currently upgrading osd daemons", 2024-04-02T05:15:59.994 INFO:teuthology.orchestra.run.smithi086.stdout: "is_paused": false 2024-04-02T05:15:59.995 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:16:00.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:00 smithi192 ceph-mon[68000]: from='client.54249 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:00.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:00 smithi192 ceph-mon[68000]: from='client.64198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:00.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:00 smithi192 ceph-mon[68000]: from='client.64204 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:00.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:00 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/516098992' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:00.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:00 smithi192 ceph-mon[68000]: from='client.54273 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:00.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:00 smithi192 ceph-mon[68000]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:16:00.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:00 smithi086 ceph-mon[86443]: from='client.54249 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:00.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:00 smithi086 ceph-mon[86443]: from='client.64198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:00.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:00 smithi086 ceph-mon[86443]: from='client.64204 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:00.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:00 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/516098992' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:00.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:00 smithi086 ceph-mon[86443]: from='client.54273 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:00.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:00 smithi086 ceph-mon[86443]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:16:00.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:00 smithi086 ceph-mon[88839]: from='client.54249 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:00.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:00 smithi086 ceph-mon[88839]: from='client.64198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:00.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:00 smithi086 ceph-mon[88839]: from='client.64204 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:00.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:00 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/516098992' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:00.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:00 smithi086 ceph-mon[88839]: from='client.54273 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:00.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:00 smithi086 ceph-mon[88839]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:16:02.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:02 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:16:02.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:02 smithi192 ceph-mon[68000]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:16:02.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:02 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-02T05:16:02.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:02 smithi192 ceph-mon[68000]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-02T05:16:02.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:02 smithi192 ceph-mon[68000]: Upgrade: osd.1 is safe to restart 2024-04-02T05:16:02.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:02 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:16:02.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:02 smithi086 ceph-mon[86443]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:16:02.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:02 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-02T05:16:02.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:02 smithi086 ceph-mon[86443]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-02T05:16:02.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:02 smithi086 ceph-mon[86443]: Upgrade: osd.1 is safe to restart 2024-04-02T05:16:02.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:02 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:16:02.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:02 smithi086 ceph-mon[88839]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:16:02.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:02 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-02T05:16:02.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:02 smithi086 ceph-mon[88839]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-02T05:16:02.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:02 smithi086 ceph-mon[88839]: Upgrade: osd.1 is safe to restart 2024-04-02T05:16:03.538 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:16:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:16:03] "GET /metrics HTTP/1.1" 200 38149 "" "Prometheus/2.43.0" 2024-04-02T05:16:05.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:04 smithi086 ceph-mon[86443]: Upgrade: Updating osd.1 2024-04-02T05:16:05.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:04 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:05.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:04 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-02T05:16:05.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:04 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:16:05.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:04 smithi086 ceph-mon[86443]: Deploying daemon osd.1 on smithi086 2024-04-02T05:16:05.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:04 smithi086 ceph-mon[86443]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:16:05.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:04 smithi086 ceph-mon[88839]: Upgrade: Updating osd.1 2024-04-02T05:16:05.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:04 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:05.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:04 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-02T05:16:05.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:04 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:16:05.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:04 smithi086 ceph-mon[88839]: Deploying daemon osd.1 on smithi086 2024-04-02T05:16:05.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:04 smithi086 ceph-mon[88839]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:16:05.381 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:04 smithi192 ceph-mon[68000]: Upgrade: Updating osd.1 2024-04-02T05:16:05.381 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:04 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:05.381 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:04 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-02T05:16:05.381 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:04 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:16:05.381 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:04 smithi192 ceph-mon[68000]: Deploying daemon osd.1 on smithi086 2024-04-02T05:16:05.381 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:04 smithi192 ceph-mon[68000]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:16:07.280 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:07 smithi086 systemd[1]: Stopping Ceph osd.1 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:16:07.541 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:07 smithi086 ceph-mon[88839]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:16:07.541 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:07 smithi086 ceph-mon[86443]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:16:07.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:07 smithi192 ceph-mon[68000]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:16:07.833 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:07 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1[35345]: 2024-04-02T05:16:07.537+0000 7f55e37bb700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T05:16:07.833 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:07 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1[35345]: 2024-04-02T05:16:07.537+0000 7f55e37bb700 -1 osd.1 116 *** Got signal Terminated *** 2024-04-02T05:16:07.833 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:07 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1[35345]: 2024-04-02T05:16:07.537+0000 7f55e37bb700 -1 osd.1 116 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T05:16:08.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:08 smithi086 ceph-mon[86443]: osd.1 marked itself down and dead 2024-04-02T05:16:08.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:08 smithi086 ceph-mon[88839]: osd.1 marked itself down and dead 2024-04-02T05:16:08.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:08 smithi192 ceph-mon[68000]: osd.1 marked itself down and dead 2024-04-02T05:16:08.958 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:08 smithi086 podman[100698]: 2024-04-02 05:16:08.671997641 +0000 UTC m=+1.247665761 container died 203689f71603aeef19ec8b900582cbdacbca10513ab85d3974edea8cc62f1124 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1, version=8, vcs-type=git, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, name=centos-stream, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, distribution-scope=public, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD) 2024-04-02T05:16:09.331 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:09 smithi086 podman[100698]: 2024-04-02 05:16:09.146433366 +0000 UTC m=+1.722101479 container cleanup 203689f71603aeef19ec8b900582cbdacbca10513ab85d3974edea8cc62f1124 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, vcs-type=git, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., io.openshift.expose-services=) 2024-04-02T05:16:09.331 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:09 smithi086 bash[100698]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1 2024-04-02T05:16:09.583 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:09 smithi086 podman[100715]: 2024-04-02 05:16:09.330897629 +0000 UTC m=+0.655112377 container remove 203689f71603aeef19ec8b900582cbdacbca10513ab85d3974edea8cc62f1124 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., io.openshift.expose-services=, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, ceph=True, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, io.buildah.version=1.19.8, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-04-02T05:16:09.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:09 smithi086 ceph-mon[88839]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:16:09.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:09 smithi086 ceph-mon[88839]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-02T05:16:09.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:09 smithi086 ceph-mon[88839]: failed to encode map e117 with expected crc 2024-04-02T05:16:09.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:09 smithi086 ceph-mon[88839]: failed to encode map e117 with expected crc 2024-04-02T05:16:09.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:09 smithi086 ceph-mon[88839]: failed to encode map e117 with expected crc 2024-04-02T05:16:09.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:09 smithi086 ceph-mon[88839]: failed to encode map e117 with expected crc 2024-04-02T05:16:09.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:09 smithi086 ceph-mon[88839]: failed to encode map e117 with expected crc 2024-04-02T05:16:09.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:09 smithi086 ceph-mon[88839]: failed to encode map e117 with expected crc 2024-04-02T05:16:09.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:09 smithi086 ceph-mon[88839]: osdmap e117: 8 total, 7 up, 8 in 2024-04-02T05:16:09.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:09 smithi086 ceph-mon[88839]: failed to encode map e117 with expected crc 2024-04-02T05:16:09.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:09 smithi086 ceph-mon[88839]: failed to encode map e117 with expected crc 2024-04-02T05:16:09.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:09 smithi086 ceph-mon[88839]: failed to encode map e117 with expected crc 2024-04-02T05:16:09.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:09 smithi086 ceph-mon[88839]: failed to encode map e117 with expected crc 2024-04-02T05:16:09.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:09 smithi086 ceph-mon[86443]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:16:09.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:09 smithi086 ceph-mon[86443]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-02T05:16:09.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:09 smithi086 ceph-mon[86443]: failed to encode map e117 with expected crc 2024-04-02T05:16:09.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:09 smithi086 ceph-mon[86443]: failed to encode map e117 with expected crc 2024-04-02T05:16:09.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:09 smithi086 ceph-mon[86443]: failed to encode map e117 with expected crc 2024-04-02T05:16:09.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:09 smithi086 ceph-mon[86443]: failed to encode map e117 with expected crc 2024-04-02T05:16:09.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:09 smithi086 ceph-mon[86443]: failed to encode map e117 with expected crc 2024-04-02T05:16:09.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:09 smithi086 ceph-mon[86443]: failed to encode map e117 with expected crc 2024-04-02T05:16:09.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:09 smithi086 ceph-mon[86443]: osdmap e117: 8 total, 7 up, 8 in 2024-04-02T05:16:09.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:09 smithi086 ceph-mon[86443]: failed to encode map e117 with expected crc 2024-04-02T05:16:09.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:09 smithi086 ceph-mon[86443]: failed to encode map e117 with expected crc 2024-04-02T05:16:09.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:09 smithi086 ceph-mon[86443]: failed to encode map e117 with expected crc 2024-04-02T05:16:09.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:09 smithi086 ceph-mon[86443]: failed to encode map e117 with expected crc 2024-04-02T05:16:09.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:09 smithi192 ceph-mon[68000]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:16:09.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:09 smithi192 ceph-mon[68000]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-02T05:16:09.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:09 smithi192 ceph-mon[68000]: failed to encode map e117 with expected crc 2024-04-02T05:16:09.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:09 smithi192 ceph-mon[68000]: failed to encode map e117 with expected crc 2024-04-02T05:16:09.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:09 smithi192 ceph-mon[68000]: failed to encode map e117 with expected crc 2024-04-02T05:16:09.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:09 smithi192 ceph-mon[68000]: failed to encode map e117 with expected crc 2024-04-02T05:16:09.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:09 smithi192 ceph-mon[68000]: failed to encode map e117 with expected crc 2024-04-02T05:16:09.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:09 smithi192 ceph-mon[68000]: failed to encode map e117 with expected crc 2024-04-02T05:16:09.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:09 smithi192 ceph-mon[68000]: osdmap e117: 8 total, 7 up, 8 in 2024-04-02T05:16:09.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:09 smithi192 ceph-mon[68000]: failed to encode map e117 with expected crc 2024-04-02T05:16:09.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:09 smithi192 ceph-mon[68000]: failed to encode map e117 with expected crc 2024-04-02T05:16:09.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:09 smithi192 ceph-mon[68000]: failed to encode map e117 with expected crc 2024-04-02T05:16:09.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:09 smithi192 ceph-mon[68000]: failed to encode map e117 with expected crc 2024-04-02T05:16:10.054 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:09 smithi086 podman[100812]: 2024-04-02 05:16:09.662491035 +0000 UTC m=+0.018366342 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:16:10.054 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:09 smithi086 podman[100812]: 2024-04-02 05:16:09.790176075 +0000 UTC m=+0.146051401 container create d07b454001918579376d6e4d9261529b717c59ce4feab9f4023041073237fa79 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-6cec793) 2024-04-02T05:16:10.333 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:10 smithi086 podman[100812]: 2024-04-02 05:16:10.051490947 +0000 UTC m=+0.407366238 container init d07b454001918579376d6e4d9261529b717c59ce4feab9f4023041073237fa79 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240326, RELEASE=squid-6cec793, ceph=True) 2024-04-02T05:16:10.333 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:10 smithi086 podman[100812]: 2024-04-02 05:16:10.055431726 +0000 UTC m=+0.411307018 container start d07b454001918579376d6e4d9261529b717c59ce4feab9f4023041073237fa79 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1-deactivate, org.label-schema.license=GPLv2, RELEASE=squid-6cec793, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-04-02T05:16:10.333 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:10 smithi086 podman[100812]: 2024-04-02 05:16:10.159460414 +0000 UTC m=+0.515335710 container attach d07b454001918579376d6e4d9261529b717c59ce4feab9f4023041073237fa79 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=squid-6cec793, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326) 2024-04-02T05:16:10.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:10 smithi192 ceph-mon[68000]: failed to encode map e118 with expected crc 2024-04-02T05:16:10.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:10 smithi192 ceph-mon[68000]: failed to encode map e118 with expected crc 2024-04-02T05:16:10.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:10 smithi192 ceph-mon[68000]: osdmap e118: 8 total, 7 up, 8 in 2024-04-02T05:16:10.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:10 smithi192 ceph-mon[68000]: failed to encode map e118 with expected crc 2024-04-02T05:16:10.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:10 smithi192 ceph-mon[68000]: failed to encode map e118 with expected crc 2024-04-02T05:16:10.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:10 smithi192 ceph-mon[68000]: failed to encode map e118 with expected crc 2024-04-02T05:16:10.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:10 smithi192 ceph-mon[68000]: failed to encode map e118 with expected crc 2024-04-02T05:16:10.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:10 smithi192 ceph-mon[68000]: failed to encode map e118 with expected crc 2024-04-02T05:16:10.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:10 smithi192 ceph-mon[68000]: failed to encode map e118 with expected crc 2024-04-02T05:16:10.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:10 smithi192 ceph-mon[68000]: failed to encode map e118 with expected crc 2024-04-02T05:16:10.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:10 smithi192 ceph-mon[68000]: failed to encode map e118 with expected crc 2024-04-02T05:16:10.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:10 smithi192 ceph-mon[68000]: failed to encode map e118 with expected crc 2024-04-02T05:16:10.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:10 smithi192 ceph-mon[68000]: failed to encode map e118 with expected crc 2024-04-02T05:16:10.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:10 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [4, 1]}]: dispatch 2024-04-02T05:16:10.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:10 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 2]}]: dispatch 2024-04-02T05:16:10.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:10 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [4, 1]}]: dispatch 2024-04-02T05:16:10.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:10 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.c", "id": [3, 1]}]: dispatch 2024-04-02T05:16:10.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:10 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 2]}]: dispatch 2024-04-02T05:16:10.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:10 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [7, 2]}]: dispatch 2024-04-02T05:16:10.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:10 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.c", "id": [3, 1]}]: dispatch 2024-04-02T05:16:10.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:10 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.11", "id": [4, 2]}]: dispatch 2024-04-02T05:16:10.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:10 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [7, 2]}]: dispatch 2024-04-02T05:16:10.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:10 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.11", "id": [4, 2]}]: dispatch 2024-04-02T05:16:10.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:10 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 5]}]: dispatch 2024-04-02T05:16:10.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:10 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 5]}]: dispatch 2024-04-02T05:16:10.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:10 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 1]}]: dispatch 2024-04-02T05:16:10.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:10 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 1]}]: dispatch 2024-04-02T05:16:10.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:10 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.11", "id": [4, 1]}]: dispatch 2024-04-02T05:16:10.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:10 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.11", "id": [4, 1]}]: dispatch 2024-04-02T05:16:10.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:10 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.15", "id": [5, 2]}]: dispatch 2024-04-02T05:16:10.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:10 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.15", "id": [5, 2]}]: dispatch 2024-04-02T05:16:10.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:10 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1a", "id": [7, 6]}]: dispatch 2024-04-02T05:16:10.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:10 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1a", "id": [7, 6]}]: dispatch 2024-04-02T05:16:10.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:10 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:10.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:10 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:16:10.685 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:10 smithi086 podman[100812]: 2024-04-02 05:16:10.367677489 +0000 UTC m=+0.723552784 container died d07b454001918579376d6e4d9261529b717c59ce4feab9f4023041073237fa79 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1-deactivate, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, RELEASE=squid-6cec793, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-04-02T05:16:10.686 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[88839]: failed to encode map e118 with expected crc 2024-04-02T05:16:10.686 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[88839]: failed to encode map e118 with expected crc 2024-04-02T05:16:10.686 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[88839]: osdmap e118: 8 total, 7 up, 8 in 2024-04-02T05:16:10.686 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[88839]: failed to encode map e118 with expected crc 2024-04-02T05:16:10.686 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[88839]: failed to encode map e118 with expected crc 2024-04-02T05:16:10.686 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[88839]: failed to encode map e118 with expected crc 2024-04-02T05:16:10.686 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[88839]: failed to encode map e118 with expected crc 2024-04-02T05:16:10.686 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[88839]: failed to encode map e118 with expected crc 2024-04-02T05:16:10.686 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[88839]: failed to encode map e118 with expected crc 2024-04-02T05:16:10.686 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[88839]: failed to encode map e118 with expected crc 2024-04-02T05:16:10.687 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[88839]: failed to encode map e118 with expected crc 2024-04-02T05:16:10.687 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[88839]: failed to encode map e118 with expected crc 2024-04-02T05:16:10.687 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[88839]: failed to encode map e118 with expected crc 2024-04-02T05:16:10.687 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [4, 1]}]: dispatch 2024-04-02T05:16:10.687 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 2]}]: dispatch 2024-04-02T05:16:10.687 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [4, 1]}]: dispatch 2024-04-02T05:16:10.687 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.c", "id": [3, 1]}]: dispatch 2024-04-02T05:16:10.687 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 2]}]: dispatch 2024-04-02T05:16:10.687 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [7, 2]}]: dispatch 2024-04-02T05:16:10.687 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.c", "id": [3, 1]}]: dispatch 2024-04-02T05:16:10.687 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.11", "id": [4, 2]}]: dispatch 2024-04-02T05:16:10.687 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [7, 2]}]: dispatch 2024-04-02T05:16:10.687 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.11", "id": [4, 2]}]: dispatch 2024-04-02T05:16:10.687 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 5]}]: dispatch 2024-04-02T05:16:10.688 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 5]}]: dispatch 2024-04-02T05:16:10.688 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 1]}]: dispatch 2024-04-02T05:16:10.688 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 1]}]: dispatch 2024-04-02T05:16:10.688 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.11", "id": [4, 1]}]: dispatch 2024-04-02T05:16:10.688 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.11", "id": [4, 1]}]: dispatch 2024-04-02T05:16:10.688 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[86443]: failed to encode map e118 with expected crc 2024-04-02T05:16:10.688 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[86443]: failed to encode map e118 with expected crc 2024-04-02T05:16:10.688 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[86443]: osdmap e118: 8 total, 7 up, 8 in 2024-04-02T05:16:10.688 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[86443]: failed to encode map e118 with expected crc 2024-04-02T05:16:10.688 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[86443]: failed to encode map e118 with expected crc 2024-04-02T05:16:10.689 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[86443]: failed to encode map e118 with expected crc 2024-04-02T05:16:10.689 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[86443]: failed to encode map e118 with expected crc 2024-04-02T05:16:10.689 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[86443]: failed to encode map e118 with expected crc 2024-04-02T05:16:10.689 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[86443]: failed to encode map e118 with expected crc 2024-04-02T05:16:10.689 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[86443]: failed to encode map e118 with expected crc 2024-04-02T05:16:10.689 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[86443]: failed to encode map e118 with expected crc 2024-04-02T05:16:10.689 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[86443]: failed to encode map e118 with expected crc 2024-04-02T05:16:10.689 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[86443]: failed to encode map e118 with expected crc 2024-04-02T05:16:10.689 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [4, 1]}]: dispatch 2024-04-02T05:16:10.689 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 2]}]: dispatch 2024-04-02T05:16:10.689 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [4, 1]}]: dispatch 2024-04-02T05:16:10.689 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.c", "id": [3, 1]}]: dispatch 2024-04-02T05:16:10.689 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 2]}]: dispatch 2024-04-02T05:16:10.689 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [7, 2]}]: dispatch 2024-04-02T05:16:10.689 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.c", "id": [3, 1]}]: dispatch 2024-04-02T05:16:10.690 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.11", "id": [4, 2]}]: dispatch 2024-04-02T05:16:10.690 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [7, 2]}]: dispatch 2024-04-02T05:16:10.690 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.11", "id": [4, 2]}]: dispatch 2024-04-02T05:16:10.690 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 5]}]: dispatch 2024-04-02T05:16:10.690 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 5]}]: dispatch 2024-04-02T05:16:10.690 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 1]}]: dispatch 2024-04-02T05:16:10.690 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 1]}]: dispatch 2024-04-02T05:16:10.690 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.11", "id": [4, 1]}]: dispatch 2024-04-02T05:16:10.690 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.11", "id": [4, 1]}]: dispatch 2024-04-02T05:16:10.690 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.15", "id": [5, 2]}]: dispatch 2024-04-02T05:16:10.690 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.15", "id": [5, 2]}]: dispatch 2024-04-02T05:16:10.691 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1a", "id": [7, 6]}]: dispatch 2024-04-02T05:16:10.691 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1a", "id": [7, 6]}]: dispatch 2024-04-02T05:16:10.691 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:10.691 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:16:10.691 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.15", "id": [5, 2]}]: dispatch 2024-04-02T05:16:10.691 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.15", "id": [5, 2]}]: dispatch 2024-04-02T05:16:10.691 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1a", "id": [7, 6]}]: dispatch 2024-04-02T05:16:10.691 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1a", "id": [7, 6]}]: dispatch 2024-04-02T05:16:10.691 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:10.691 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:10 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:16:11.333 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:11 smithi086 podman[100838]: 2024-04-02 05:16:11.003010887 +0000 UTC m=+0.627911008 container remove d07b454001918579376d6e4d9261529b717c59ce4feab9f4023041073237fa79 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1-deactivate, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=squid-6cec793, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-04-02T05:16:11.333 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:11 smithi086 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.1.service: Deactivated successfully. 2024-04-02T05:16:11.333 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:11 smithi086 systemd[1]: Stopped Ceph osd.1 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:16:11.333 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:11 smithi086 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.1.service: Consumed 14.355s CPU time. 2024-04-02T05:16:11.333 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:11 smithi086 systemd[1]: Starting Ceph osd.1 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:16:11.610 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:11 smithi086 podman[100940]: 2024-04-02 05:16:11.358058403 +0000 UTC m=+0.021866515 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:16:11.610 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:11 smithi086 podman[100940]: 2024-04-02 05:16:11.487178722 +0000 UTC m=+0.150986818 container create f416049fab46b0ec16892f79f3999c9898217e90d115643c6bf408a8bf5cccc3 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1-activate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, RELEASE=squid-6cec793, org.label-schema.build-date=20240326, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux ) 2024-04-02T05:16:11.610 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:11 smithi086 ceph-mon[86443]: pgmap v70: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-02T05:16:11.610 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:11 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [4, 1]}]': finished 2024-04-02T05:16:11.610 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:11 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 2]}]': finished 2024-04-02T05:16:11.610 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:11 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.c", "id": [3, 1]}]': finished 2024-04-02T05:16:11.611 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:11 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [7, 2]}]': finished 2024-04-02T05:16:11.611 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:11 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.11", "id": [4, 2]}]': finished 2024-04-02T05:16:11.611 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:11 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 5]}]': finished 2024-04-02T05:16:11.611 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:11 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 1]}]': finished 2024-04-02T05:16:11.611 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:11 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.11", "id": [4, 1]}]': finished 2024-04-02T05:16:11.611 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:11 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.15", "id": [5, 2]}]': finished 2024-04-02T05:16:11.611 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:11 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1a", "id": [7, 6]}]': finished 2024-04-02T05:16:11.611 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:11 smithi086 ceph-mon[86443]: failed to encode map e119 with expected crc 2024-04-02T05:16:11.611 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:11 smithi086 ceph-mon[86443]: failed to encode map e119 with expected crc 2024-04-02T05:16:11.611 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:11 smithi086 ceph-mon[86443]: osdmap e119: 8 total, 7 up, 8 in 2024-04-02T05:16:11.611 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:11 smithi086 ceph-mon[86443]: failed to encode map e119 with expected crc 2024-04-02T05:16:11.611 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:11 smithi086 ceph-mon[86443]: failed to encode map e119 with expected crc 2024-04-02T05:16:11.611 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:11 smithi086 ceph-mon[86443]: failed to encode map e119 with expected crc 2024-04-02T05:16:11.612 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:11 smithi086 ceph-mon[88839]: pgmap v70: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-02T05:16:11.612 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:11 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [4, 1]}]': finished 2024-04-02T05:16:11.612 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:11 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 2]}]': finished 2024-04-02T05:16:11.612 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:11 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.c", "id": [3, 1]}]': finished 2024-04-02T05:16:11.612 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:11 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [7, 2]}]': finished 2024-04-02T05:16:11.612 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:11 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.11", "id": [4, 2]}]': finished 2024-04-02T05:16:11.612 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:11 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 5]}]': finished 2024-04-02T05:16:11.612 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:11 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 1]}]': finished 2024-04-02T05:16:11.612 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:11 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.11", "id": [4, 1]}]': finished 2024-04-02T05:16:11.612 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:11 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.15", "id": [5, 2]}]': finished 2024-04-02T05:16:11.612 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:11 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1a", "id": [7, 6]}]': finished 2024-04-02T05:16:11.612 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:11 smithi086 ceph-mon[88839]: failed to encode map e119 with expected crc 2024-04-02T05:16:11.612 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:11 smithi086 ceph-mon[88839]: failed to encode map e119 with expected crc 2024-04-02T05:16:11.613 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:11 smithi086 ceph-mon[88839]: osdmap e119: 8 total, 7 up, 8 in 2024-04-02T05:16:11.613 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:11 smithi086 ceph-mon[88839]: failed to encode map e119 with expected crc 2024-04-02T05:16:11.613 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:11 smithi086 ceph-mon[88839]: failed to encode map e119 with expected crc 2024-04-02T05:16:11.613 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:11 smithi086 ceph-mon[88839]: failed to encode map e119 with expected crc 2024-04-02T05:16:11.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:11 smithi192 ceph-mon[68000]: pgmap v70: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-02T05:16:11.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:11 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.9", "id": [4, 1]}]': finished 2024-04-02T05:16:11.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:11 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 2]}]': finished 2024-04-02T05:16:11.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:11 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.c", "id": [3, 1]}]': finished 2024-04-02T05:16:11.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:11 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [7, 2]}]': finished 2024-04-02T05:16:11.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:11 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.11", "id": [4, 2]}]': finished 2024-04-02T05:16:11.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:11 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 5]}]': finished 2024-04-02T05:16:11.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:11 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 1]}]': finished 2024-04-02T05:16:11.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:11 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.11", "id": [4, 1]}]': finished 2024-04-02T05:16:11.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:11 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.15", "id": [5, 2]}]': finished 2024-04-02T05:16:11.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:11 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "5.1a", "id": [7, 6]}]': finished 2024-04-02T05:16:11.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:11 smithi192 ceph-mon[68000]: failed to encode map e119 with expected crc 2024-04-02T05:16:11.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:11 smithi192 ceph-mon[68000]: failed to encode map e119 with expected crc 2024-04-02T05:16:11.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:11 smithi192 ceph-mon[68000]: osdmap e119: 8 total, 7 up, 8 in 2024-04-02T05:16:11.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:11 smithi192 ceph-mon[68000]: failed to encode map e119 with expected crc 2024-04-02T05:16:11.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:11 smithi192 ceph-mon[68000]: failed to encode map e119 with expected crc 2024-04-02T05:16:11.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:11 smithi192 ceph-mon[68000]: failed to encode map e119 with expected crc 2024-04-02T05:16:12.083 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:11 smithi086 podman[100940]: 2024-04-02 05:16:11.714604129 +0000 UTC m=+0.378412225 container init f416049fab46b0ec16892f79f3999c9898217e90d115643c6bf408a8bf5cccc3 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1-activate, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=squid-6cec793, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-04-02T05:16:12.083 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:11 smithi086 podman[100940]: 2024-04-02 05:16:11.718121116 +0000 UTC m=+0.381929221 container start f416049fab46b0ec16892f79f3999c9898217e90d115643c6bf408a8bf5cccc3 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1-activate, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, RELEASE=squid-6cec793, GIT_CLEAN=True) 2024-04-02T05:16:12.083 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:11 smithi086 podman[100940]: 2024-04-02 05:16:11.814670625 +0000 UTC m=+0.478478722 container attach f416049fab46b0ec16892f79f3999c9898217e90d115643c6bf408a8bf5cccc3 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1-activate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-6cec793, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2) 2024-04-02T05:16:12.553 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:12 smithi086 ceph-mon[88839]: failed to encode map e120 with expected crc 2024-04-02T05:16:12.553 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:12 smithi086 ceph-mon[88839]: osdmap e120: 8 total, 7 up, 8 in 2024-04-02T05:16:12.553 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:12 smithi086 ceph-mon[88839]: failed to encode map e120 with expected crc 2024-04-02T05:16:12.553 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:12 smithi086 ceph-mon[88839]: failed to encode map e120 with expected crc 2024-04-02T05:16:12.553 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:12 smithi086 ceph-mon[88839]: failed to encode map e120 with expected crc 2024-04-02T05:16:12.553 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:12 smithi086 ceph-mon[88839]: failed to encode map e120 with expected crc 2024-04-02T05:16:12.553 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:12 smithi086 ceph-mon[88839]: failed to encode map e120 with expected crc 2024-04-02T05:16:12.554 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:12 smithi086 ceph-mon[88839]: failed to encode map e120 with expected crc 2024-04-02T05:16:12.554 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:12 smithi086 ceph-mon[88839]: failed to encode map e120 with expected crc 2024-04-02T05:16:12.554 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:12 smithi086 ceph-mon[86443]: failed to encode map e120 with expected crc 2024-04-02T05:16:12.554 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:12 smithi086 ceph-mon[86443]: osdmap e120: 8 total, 7 up, 8 in 2024-04-02T05:16:12.554 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:12 smithi086 ceph-mon[86443]: failed to encode map e120 with expected crc 2024-04-02T05:16:12.554 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:12 smithi086 ceph-mon[86443]: failed to encode map e120 with expected crc 2024-04-02T05:16:12.554 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:12 smithi086 ceph-mon[86443]: failed to encode map e120 with expected crc 2024-04-02T05:16:12.554 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:12 smithi086 ceph-mon[86443]: failed to encode map e120 with expected crc 2024-04-02T05:16:12.554 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:12 smithi086 ceph-mon[86443]: failed to encode map e120 with expected crc 2024-04-02T05:16:12.555 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:12 smithi086 ceph-mon[86443]: failed to encode map e120 with expected crc 2024-04-02T05:16:12.555 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:12 smithi086 ceph-mon[86443]: failed to encode map e120 with expected crc 2024-04-02T05:16:12.555 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:12 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1-activate[100955]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-02T05:16:12.555 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:12 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1-activate[100955]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-02T05:16:12.555 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:12 smithi086 bash[100940]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-02T05:16:12.555 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:12 smithi086 bash[100940]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-02T05:16:12.555 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:12 smithi086 bash[100940]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-02T05:16:12.555 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:12 smithi086 bash[100940]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-02T05:16:12.555 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:12 smithi086 bash[100940]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-02T05:16:12.555 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:12 smithi086 bash[100940]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-02T05:16:12.555 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:12 smithi086 bash[100940]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-02T05:16:12.555 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:12 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1-activate[100955]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-02T05:16:12.555 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:12 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1-activate[100955]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-02T05:16:12.556 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:12 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1-activate[100955]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-02T05:16:12.556 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:12 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1-activate[100955]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-02T05:16:12.556 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:12 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1-activate[100955]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-02T05:16:12.556 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:12 smithi086 podman[100940]: 2024-04-02 05:16:12.19018453 +0000 UTC m=+0.853992629 container died f416049fab46b0ec16892f79f3999c9898217e90d115643c6bf408a8bf5cccc3 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240326, RELEASE=squid-6cec793, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5) 2024-04-02T05:16:12.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:12 smithi192 ceph-mon[68000]: failed to encode map e120 with expected crc 2024-04-02T05:16:12.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:12 smithi192 ceph-mon[68000]: osdmap e120: 8 total, 7 up, 8 in 2024-04-02T05:16:12.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:12 smithi192 ceph-mon[68000]: failed to encode map e120 with expected crc 2024-04-02T05:16:12.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:12 smithi192 ceph-mon[68000]: failed to encode map e120 with expected crc 2024-04-02T05:16:12.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:12 smithi192 ceph-mon[68000]: failed to encode map e120 with expected crc 2024-04-02T05:16:12.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:12 smithi192 ceph-mon[68000]: failed to encode map e120 with expected crc 2024-04-02T05:16:12.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:12 smithi192 ceph-mon[68000]: failed to encode map e120 with expected crc 2024-04-02T05:16:12.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:12 smithi192 ceph-mon[68000]: failed to encode map e120 with expected crc 2024-04-02T05:16:12.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:12 smithi192 ceph-mon[68000]: failed to encode map e120 with expected crc 2024-04-02T05:16:13.155 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:12 smithi086 podman[101091]: 2024-04-02 05:16:12.893620682 +0000 UTC m=+0.697923396 container remove f416049fab46b0ec16892f79f3999c9898217e90d115643c6bf408a8bf5cccc3 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1-activate, RELEASE=squid-6cec793, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-04-02T05:16:13.156 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:13 smithi086 podman[101154]: 2024-04-02 05:16:13.007955749 +0000 UTC m=+0.018261069 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:16:13.457 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:16:13 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:16:13] "GET /metrics HTTP/1.1" 200 38153 "" "Prometheus/2.43.0" 2024-04-02T05:16:13.458 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:13 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:16:13.458 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:13 smithi086 ceph-mon[86443]: pgmap v73: 161 pgs: 1 remapped+peering, 5 peering, 19 active+undersized+degraded, 39 active+undersized, 97 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 80/723 objects degraded (11.065%) 2024-04-02T05:16:13.458 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:13 smithi086 ceph-mon[86443]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-02T05:16:13.458 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:13 smithi086 ceph-mon[86443]: Health check failed: Degraded data redundancy: 80/723 objects degraded (11.065%), 19 pgs degraded (PG_DEGRADED) 2024-04-02T05:16:13.458 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:13 smithi086 ceph-mon[86443]: failed to encode map e121 with expected crc 2024-04-02T05:16:13.458 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:13 smithi086 ceph-mon[86443]: failed to encode map e121 with expected crc 2024-04-02T05:16:13.458 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:13 smithi086 ceph-mon[86443]: failed to encode map e121 with expected crc 2024-04-02T05:16:13.458 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:13 smithi086 ceph-mon[86443]: failed to encode map e121 with expected crc 2024-04-02T05:16:13.458 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:13 smithi086 ceph-mon[86443]: failed to encode map e121 with expected crc 2024-04-02T05:16:13.458 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:13 smithi086 ceph-mon[86443]: osdmap e121: 8 total, 7 up, 8 in 2024-04-02T05:16:13.458 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:13 smithi086 ceph-mon[86443]: failed to encode map e121 with expected crc 2024-04-02T05:16:13.458 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:13 smithi086 ceph-mon[86443]: failed to encode map e121 with expected crc 2024-04-02T05:16:13.458 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:13 smithi086 ceph-mon[86443]: failed to encode map e121 with expected crc 2024-04-02T05:16:13.459 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:13 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:16:13.459 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:13 smithi086 ceph-mon[88839]: pgmap v73: 161 pgs: 1 remapped+peering, 5 peering, 19 active+undersized+degraded, 39 active+undersized, 97 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 80/723 objects degraded (11.065%) 2024-04-02T05:16:13.459 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:13 smithi086 ceph-mon[88839]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-02T05:16:13.459 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:13 smithi086 ceph-mon[88839]: Health check failed: Degraded data redundancy: 80/723 objects degraded (11.065%), 19 pgs degraded (PG_DEGRADED) 2024-04-02T05:16:13.459 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:13 smithi086 ceph-mon[88839]: failed to encode map e121 with expected crc 2024-04-02T05:16:13.459 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:13 smithi086 ceph-mon[88839]: failed to encode map e121 with expected crc 2024-04-02T05:16:13.459 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:13 smithi086 ceph-mon[88839]: failed to encode map e121 with expected crc 2024-04-02T05:16:13.459 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:13 smithi086 ceph-mon[88839]: failed to encode map e121 with expected crc 2024-04-02T05:16:13.459 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:13 smithi086 ceph-mon[88839]: failed to encode map e121 with expected crc 2024-04-02T05:16:13.459 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:13 smithi086 ceph-mon[88839]: osdmap e121: 8 total, 7 up, 8 in 2024-04-02T05:16:13.459 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:13 smithi086 ceph-mon[88839]: failed to encode map e121 with expected crc 2024-04-02T05:16:13.460 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:13 smithi086 ceph-mon[88839]: failed to encode map e121 with expected crc 2024-04-02T05:16:13.460 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:13 smithi086 ceph-mon[88839]: failed to encode map e121 with expected crc 2024-04-02T05:16:13.460 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:13 smithi086 podman[101154]: 2024-04-02 05:16:13.153025872 +0000 UTC m=+0.163331202 container create 76fe72844771885bdefd6d204dc4bf7836f688969f0bddcdd30ddf656ce146b7 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1, maintainer=Guillaume Abrioux , RELEASE=squid-6cec793, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2) 2024-04-02T05:16:13.460 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:13 smithi086 podman[101154]: 2024-04-02 05:16:13.355530543 +0000 UTC m=+0.365835867 container init 76fe72844771885bdefd6d204dc4bf7836f688969f0bddcdd30ddf656ce146b7 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-6cec793, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-04-02T05:16:13.460 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:13 smithi086 podman[101154]: 2024-04-02 05:16:13.358942387 +0000 UTC m=+0.369247707 container start 76fe72844771885bdefd6d204dc4bf7836f688969f0bddcdd30ddf656ce146b7 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1, ceph=True, GIT_BRANCH=HEAD, RELEASE=squid-6cec793, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-02T05:16:13.460 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:13 smithi086 bash[101154]: 76fe72844771885bdefd6d204dc4bf7836f688969f0bddcdd30ddf656ce146b7 2024-04-02T05:16:13.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:13 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:16:13.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:13 smithi192 ceph-mon[68000]: pgmap v73: 161 pgs: 1 remapped+peering, 5 peering, 19 active+undersized+degraded, 39 active+undersized, 97 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 80/723 objects degraded (11.065%) 2024-04-02T05:16:13.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:13 smithi192 ceph-mon[68000]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-02T05:16:13.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:13 smithi192 ceph-mon[68000]: Health check failed: Degraded data redundancy: 80/723 objects degraded (11.065%), 19 pgs degraded (PG_DEGRADED) 2024-04-02T05:16:13.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:13 smithi192 ceph-mon[68000]: failed to encode map e121 with expected crc 2024-04-02T05:16:13.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:13 smithi192 ceph-mon[68000]: failed to encode map e121 with expected crc 2024-04-02T05:16:13.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:13 smithi192 ceph-mon[68000]: failed to encode map e121 with expected crc 2024-04-02T05:16:13.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:13 smithi192 ceph-mon[68000]: failed to encode map e121 with expected crc 2024-04-02T05:16:13.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:13 smithi192 ceph-mon[68000]: failed to encode map e121 with expected crc 2024-04-02T05:16:13.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:13 smithi192 ceph-mon[68000]: osdmap e121: 8 total, 7 up, 8 in 2024-04-02T05:16:13.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:13 smithi192 ceph-mon[68000]: failed to encode map e121 with expected crc 2024-04-02T05:16:13.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:13 smithi192 ceph-mon[68000]: failed to encode map e121 with expected crc 2024-04-02T05:16:13.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:13 smithi192 ceph-mon[68000]: failed to encode map e121 with expected crc 2024-04-02T05:16:13.832 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:13 smithi086 systemd[1]: Started Ceph osd.1 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:16:14.333 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:13 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1[101169]: 2024-04-02T05:16:13.901+0000 7fbd731567c0 -1 Falling back to public interface 2024-04-02T05:16:14.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:14 smithi192 ceph-mon[68000]: failed to encode map e122 with expected crc 2024-04-02T05:16:14.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:14 smithi192 ceph-mon[68000]: failed to encode map e122 with expected crc 2024-04-02T05:16:14.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:14 smithi192 ceph-mon[68000]: osdmap e122: 8 total, 7 up, 8 in 2024-04-02T05:16:14.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:14 smithi192 ceph-mon[68000]: failed to encode map e122 with expected crc 2024-04-02T05:16:14.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:14 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:14.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:14 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:14.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:14 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:14.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:14 smithi192 ceph-mon[68000]: pgmap v76: 161 pgs: 1 remapped+peering, 5 peering, 19 active+undersized+degraded, 39 active+undersized, 97 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 80/723 objects degraded (11.065%) 2024-04-02T05:16:14.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:14 smithi086 ceph-mon[86443]: failed to encode map e122 with expected crc 2024-04-02T05:16:14.891 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:14 smithi086 ceph-mon[86443]: failed to encode map e122 with expected crc 2024-04-02T05:16:14.891 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:14 smithi086 ceph-mon[86443]: osdmap e122: 8 total, 7 up, 8 in 2024-04-02T05:16:14.892 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:14 smithi086 ceph-mon[86443]: failed to encode map e122 with expected crc 2024-04-02T05:16:14.892 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:14 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:14.892 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:14 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:14.893 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:14 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:14.893 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:14 smithi086 ceph-mon[86443]: pgmap v76: 161 pgs: 1 remapped+peering, 5 peering, 19 active+undersized+degraded, 39 active+undersized, 97 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 80/723 objects degraded (11.065%) 2024-04-02T05:16:14.893 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:14 smithi086 ceph-mon[88839]: failed to encode map e122 with expected crc 2024-04-02T05:16:14.894 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:14 smithi086 ceph-mon[88839]: failed to encode map e122 with expected crc 2024-04-02T05:16:14.894 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:14 smithi086 ceph-mon[88839]: osdmap e122: 8 total, 7 up, 8 in 2024-04-02T05:16:14.894 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:14 smithi086 ceph-mon[88839]: failed to encode map e122 with expected crc 2024-04-02T05:16:14.895 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:14 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:14.895 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:14 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:14.896 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:14 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:14.896 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:14 smithi086 ceph-mon[88839]: pgmap v76: 161 pgs: 1 remapped+peering, 5 peering, 19 active+undersized+degraded, 39 active+undersized, 97 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 80/723 objects degraded (11.065%) 2024-04-02T05:16:15.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:15 smithi086 ceph-mon[88839]: failed to encode map e123 with expected crc 2024-04-02T05:16:15.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:15 smithi086 ceph-mon[88839]: failed to encode map e123 with expected crc 2024-04-02T05:16:15.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:15 smithi086 ceph-mon[88839]: osdmap e123: 8 total, 7 up, 8 in 2024-04-02T05:16:15.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:15 smithi086 ceph-mon[88839]: failed to encode map e123 with expected crc 2024-04-02T05:16:15.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:15 smithi086 ceph-mon[86443]: failed to encode map e123 with expected crc 2024-04-02T05:16:15.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:15 smithi086 ceph-mon[86443]: failed to encode map e123 with expected crc 2024-04-02T05:16:15.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:15 smithi086 ceph-mon[86443]: osdmap e123: 8 total, 7 up, 8 in 2024-04-02T05:16:15.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:15 smithi086 ceph-mon[86443]: failed to encode map e123 with expected crc 2024-04-02T05:16:15.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:15 smithi192 ceph-mon[68000]: failed to encode map e123 with expected crc 2024-04-02T05:16:15.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:15 smithi192 ceph-mon[68000]: failed to encode map e123 with expected crc 2024-04-02T05:16:15.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:15 smithi192 ceph-mon[68000]: osdmap e123: 8 total, 7 up, 8 in 2024-04-02T05:16:15.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:15 smithi192 ceph-mon[68000]: failed to encode map e123 with expected crc 2024-04-02T05:16:16.641 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:16 smithi086 ceph-mon[88839]: pgmap v78: 161 pgs: 1 remapped+peering, 5 peering, 19 active+undersized+degraded, 39 active+undersized, 97 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 80/723 objects degraded (11.065%) 2024-04-02T05:16:16.642 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:16 smithi086 ceph-mon[86443]: pgmap v78: 161 pgs: 1 remapped+peering, 5 peering, 19 active+undersized+degraded, 39 active+undersized, 97 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 80/723 objects degraded (11.065%) 2024-04-02T05:16:16.642 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:16 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1[101169]: 2024-04-02T05:16:16.612+0000 7fbd731567c0 -1 osd.1 0 read_superblock omap replica is missing. 2024-04-02T05:16:16.642 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:16 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1[101169]: 2024-04-02T05:16:16.639+0000 7fbd731567c0 -1 osd.1 116 log_to_monitors true 2024-04-02T05:16:16.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:16 smithi192 ceph-mon[68000]: pgmap v78: 161 pgs: 1 remapped+peering, 5 peering, 19 active+undersized+degraded, 39 active+undersized, 97 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 80/723 objects degraded (11.065%) 2024-04-02T05:16:17.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:17 smithi086 ceph-mon[88839]: from='osd.1 [v2:172.21.15.86:6810/3450551821,v1:172.21.15.86:6811/3450551821]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-02T05:16:17.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:17 smithi086 ceph-mon[88839]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-02T05:16:17.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:17 smithi086 ceph-mon[86443]: from='osd.1 [v2:172.21.15.86:6810/3450551821,v1:172.21.15.86:6811/3450551821]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-02T05:16:17.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:17 smithi086 ceph-mon[86443]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-02T05:16:17.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:17 smithi192 ceph-mon[68000]: from='osd.1 [v2:172.21.15.86:6810/3450551821,v1:172.21.15.86:6811/3450551821]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-02T05:16:17.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:17 smithi192 ceph-mon[68000]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-02T05:16:18.333 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:16:17 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1[101169]: 2024-04-02T05:16:17.867+0000 7fbd605a6700 -1 osd.1 116 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T05:16:18.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:18 smithi192 ceph-mon[68000]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-02T05:16:18.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:18 smithi192 ceph-mon[68000]: failed to encode map e124 with expected crc 2024-04-02T05:16:18.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:18 smithi192 ceph-mon[68000]: failed to encode map e124 with expected crc 2024-04-02T05:16:18.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:18 smithi192 ceph-mon[68000]: from='osd.1 [v2:172.21.15.86:6810/3450551821,v1:172.21.15.86:6811/3450551821]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T05:16:18.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:18 smithi192 ceph-mon[68000]: osdmap e124: 8 total, 7 up, 8 in 2024-04-02T05:16:18.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:18 smithi192 ceph-mon[68000]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T05:16:18.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:18 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:18.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:18 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:18.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:18 smithi192 ceph-mon[68000]: from='osd.1 ' entity='osd.1' 2024-04-02T05:16:18.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:18 smithi192 ceph-mon[68000]: failed to encode map e124 with expected crc 2024-04-02T05:16:18.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:18 smithi192 ceph-mon[68000]: pgmap v80: 161 pgs: 9 active+clean+remapped, 19 active+undersized+degraded, 39 active+undersized, 94 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 519 B/s rd, 0 op/s; 80/723 objects degraded (11.065%); 37/723 objects misplaced (5.118%); 74 B/s, 4 objects/s recovering 2024-04-02T05:16:18.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:18 smithi086 ceph-mon[86443]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-02T05:16:18.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:18 smithi086 ceph-mon[86443]: failed to encode map e124 with expected crc 2024-04-02T05:16:18.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:18 smithi086 ceph-mon[86443]: failed to encode map e124 with expected crc 2024-04-02T05:16:18.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:18 smithi086 ceph-mon[86443]: from='osd.1 [v2:172.21.15.86:6810/3450551821,v1:172.21.15.86:6811/3450551821]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T05:16:18.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:18 smithi086 ceph-mon[86443]: osdmap e124: 8 total, 7 up, 8 in 2024-04-02T05:16:18.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:18 smithi086 ceph-mon[86443]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T05:16:18.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:18 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:18.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:18 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:18.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:18 smithi086 ceph-mon[86443]: from='osd.1 ' entity='osd.1' 2024-04-02T05:16:18.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:18 smithi086 ceph-mon[86443]: failed to encode map e124 with expected crc 2024-04-02T05:16:18.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:18 smithi086 ceph-mon[86443]: pgmap v80: 161 pgs: 9 active+clean+remapped, 19 active+undersized+degraded, 39 active+undersized, 94 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 519 B/s rd, 0 op/s; 80/723 objects degraded (11.065%); 37/723 objects misplaced (5.118%); 74 B/s, 4 objects/s recovering 2024-04-02T05:16:18.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:18 smithi086 ceph-mon[88839]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-02T05:16:18.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:18 smithi086 ceph-mon[88839]: failed to encode map e124 with expected crc 2024-04-02T05:16:18.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:18 smithi086 ceph-mon[88839]: failed to encode map e124 with expected crc 2024-04-02T05:16:18.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:18 smithi086 ceph-mon[88839]: from='osd.1 [v2:172.21.15.86:6810/3450551821,v1:172.21.15.86:6811/3450551821]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T05:16:18.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:18 smithi086 ceph-mon[88839]: osdmap e124: 8 total, 7 up, 8 in 2024-04-02T05:16:18.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:18 smithi086 ceph-mon[88839]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T05:16:18.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:18 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:18.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:18 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:18.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:18 smithi086 ceph-mon[88839]: from='osd.1 ' entity='osd.1' 2024-04-02T05:16:18.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:18 smithi086 ceph-mon[88839]: failed to encode map e124 with expected crc 2024-04-02T05:16:18.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:18 smithi086 ceph-mon[88839]: pgmap v80: 161 pgs: 9 active+clean+remapped, 19 active+undersized+degraded, 39 active+undersized, 94 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 519 B/s rd, 0 op/s; 80/723 objects degraded (11.065%); 37/723 objects misplaced (5.118%); 74 B/s, 4 objects/s recovering 2024-04-02T05:16:19.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:19 smithi192 ceph-mon[68000]: failed to encode map e124 with expected crc 2024-04-02T05:16:19.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:19 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:19.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:19 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:19.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:19 smithi192 ceph-mon[68000]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-02T05:16:19.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:19 smithi192 ceph-mon[68000]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-02T05:16:19.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:19 smithi192 ceph-mon[68000]: failed to encode map e125 with expected crc 2024-04-02T05:16:19.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:19 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T05:16:19.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:19 smithi192 ceph-mon[68000]: osd.1 [v2:172.21.15.86:6810/3450551821,v1:172.21.15.86:6811/3450551821] boot 2024-04-02T05:16:19.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:19 smithi192 ceph-mon[68000]: osdmap e125: 8 total, 8 up, 8 in 2024-04-02T05:16:19.712 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:19 smithi086 ceph-mon[88839]: failed to encode map e124 with expected crc 2024-04-02T05:16:19.712 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:19 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:19.712 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:19 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:19.712 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:19 smithi086 ceph-mon[88839]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-02T05:16:19.712 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:19 smithi086 ceph-mon[88839]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-02T05:16:19.712 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:19 smithi086 ceph-mon[88839]: failed to encode map e125 with expected crc 2024-04-02T05:16:19.712 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:19 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T05:16:19.712 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:19 smithi086 ceph-mon[88839]: osd.1 [v2:172.21.15.86:6810/3450551821,v1:172.21.15.86:6811/3450551821] boot 2024-04-02T05:16:19.712 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:19 smithi086 ceph-mon[88839]: osdmap e125: 8 total, 8 up, 8 in 2024-04-02T05:16:19.713 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:19 smithi086 ceph-mon[86443]: failed to encode map e124 with expected crc 2024-04-02T05:16:19.713 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:19 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:19.713 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:19 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:19.713 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:19 smithi086 ceph-mon[86443]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-02T05:16:19.713 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:19 smithi086 ceph-mon[86443]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-02T05:16:19.713 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:19 smithi086 ceph-mon[86443]: failed to encode map e125 with expected crc 2024-04-02T05:16:19.713 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:19 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T05:16:19.713 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:19 smithi086 ceph-mon[86443]: osd.1 [v2:172.21.15.86:6810/3450551821,v1:172.21.15.86:6811/3450551821] boot 2024-04-02T05:16:19.713 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:19 smithi086 ceph-mon[86443]: osdmap e125: 8 total, 8 up, 8 in 2024-04-02T05:16:20.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:20 smithi192 ceph-mon[68000]: failed to encode map e125 with expected crc 2024-04-02T05:16:20.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:20 smithi192 ceph-mon[68000]: failed to encode map e126 with expected crc 2024-04-02T05:16:20.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:20 smithi192 ceph-mon[68000]: failed to encode map e126 with expected crc 2024-04-02T05:16:20.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:20 smithi192 ceph-mon[68000]: osdmap e126: 8 total, 8 up, 8 in 2024-04-02T05:16:20.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:20 smithi192 ceph-mon[68000]: failed to encode map e126 with expected crc 2024-04-02T05:16:20.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:20 smithi192 ceph-mon[68000]: failed to encode map e126 with expected crc 2024-04-02T05:16:20.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:20 smithi192 ceph-mon[68000]: failed to encode map e126 with expected crc 2024-04-02T05:16:20.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:20 smithi192 ceph-mon[68000]: failed to encode map e126 with expected crc 2024-04-02T05:16:20.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:20 smithi192 ceph-mon[68000]: pgmap v83: 161 pgs: 9 active+clean+remapped, 19 active+undersized+degraded, 39 active+undersized, 94 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 520 B/s rd, 0 op/s; 80/723 objects degraded (11.065%); 37/723 objects misplaced (5.118%); 74 B/s, 4 objects/s recovering 2024-04-02T05:16:20.727 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:20 smithi086 ceph-mon[88839]: failed to encode map e125 with expected crc 2024-04-02T05:16:20.727 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:20 smithi086 ceph-mon[88839]: failed to encode map e126 with expected crc 2024-04-02T05:16:20.727 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:20 smithi086 ceph-mon[88839]: failed to encode map e126 with expected crc 2024-04-02T05:16:20.727 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:20 smithi086 ceph-mon[88839]: osdmap e126: 8 total, 8 up, 8 in 2024-04-02T05:16:20.727 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:20 smithi086 ceph-mon[88839]: failed to encode map e126 with expected crc 2024-04-02T05:16:20.727 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:20 smithi086 ceph-mon[88839]: failed to encode map e126 with expected crc 2024-04-02T05:16:20.728 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:20 smithi086 ceph-mon[88839]: failed to encode map e126 with expected crc 2024-04-02T05:16:20.728 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:20 smithi086 ceph-mon[88839]: failed to encode map e126 with expected crc 2024-04-02T05:16:20.728 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:20 smithi086 ceph-mon[88839]: pgmap v83: 161 pgs: 9 active+clean+remapped, 19 active+undersized+degraded, 39 active+undersized, 94 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 520 B/s rd, 0 op/s; 80/723 objects degraded (11.065%); 37/723 objects misplaced (5.118%); 74 B/s, 4 objects/s recovering 2024-04-02T05:16:20.728 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:20 smithi086 ceph-mon[86443]: failed to encode map e125 with expected crc 2024-04-02T05:16:20.728 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:20 smithi086 ceph-mon[86443]: failed to encode map e126 with expected crc 2024-04-02T05:16:20.728 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:20 smithi086 ceph-mon[86443]: failed to encode map e126 with expected crc 2024-04-02T05:16:20.728 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:20 smithi086 ceph-mon[86443]: osdmap e126: 8 total, 8 up, 8 in 2024-04-02T05:16:20.728 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:20 smithi086 ceph-mon[86443]: failed to encode map e126 with expected crc 2024-04-02T05:16:20.728 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:20 smithi086 ceph-mon[86443]: failed to encode map e126 with expected crc 2024-04-02T05:16:20.728 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:20 smithi086 ceph-mon[86443]: failed to encode map e126 with expected crc 2024-04-02T05:16:20.728 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:20 smithi086 ceph-mon[86443]: failed to encode map e126 with expected crc 2024-04-02T05:16:20.728 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:20 smithi086 ceph-mon[86443]: pgmap v83: 161 pgs: 9 active+clean+remapped, 19 active+undersized+degraded, 39 active+undersized, 94 active+clean; 585 KiB data, 137 MiB used, 715 GiB / 715 GiB avail; 520 B/s rd, 0 op/s; 80/723 objects degraded (11.065%); 37/723 objects misplaced (5.118%); 74 B/s, 4 objects/s recovering 2024-04-02T05:16:21.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:21 smithi192 ceph-mon[68000]: failed to encode map e126 with expected crc 2024-04-02T05:16:21.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:21 smithi192 ceph-mon[68000]: failed to encode map e126 with expected crc 2024-04-02T05:16:21.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:21 smithi192 ceph-mon[68000]: failed to encode map e126 with expected crc 2024-04-02T05:16:21.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:21 smithi192 ceph-mon[68000]: failed to encode map e126 with expected crc 2024-04-02T05:16:21.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:21 smithi192 ceph-mon[68000]: failed to encode map e126 with expected crc 2024-04-02T05:16:21.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:21 smithi192 ceph-mon[68000]: failed to encode map e126 with expected crc 2024-04-02T05:16:21.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:21 smithi192 ceph-mon[68000]: failed to encode map e127 with expected crc 2024-04-02T05:16:21.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:21 smithi192 ceph-mon[68000]: failed to encode map e127 with expected crc 2024-04-02T05:16:21.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:21 smithi192 ceph-mon[68000]: failed to encode map e127 with expected crc 2024-04-02T05:16:21.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:21 smithi192 ceph-mon[68000]: failed to encode map e127 with expected crc 2024-04-02T05:16:21.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:21 smithi192 ceph-mon[68000]: osdmap e127: 8 total, 8 up, 8 in 2024-04-02T05:16:21.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:21 smithi192 ceph-mon[68000]: failed to encode map e127 with expected crc 2024-04-02T05:16:21.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:21 smithi192 ceph-mon[68000]: failed to encode map e127 with expected crc 2024-04-02T05:16:21.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:21 smithi192 ceph-mon[68000]: failed to encode map e127 with expected crc 2024-04-02T05:16:21.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:21 smithi192 ceph-mon[68000]: failed to encode map e127 with expected crc 2024-04-02T05:16:21.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:21 smithi086 ceph-mon[88839]: failed to encode map e126 with expected crc 2024-04-02T05:16:21.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:21 smithi086 ceph-mon[88839]: failed to encode map e126 with expected crc 2024-04-02T05:16:21.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:21 smithi086 ceph-mon[88839]: failed to encode map e126 with expected crc 2024-04-02T05:16:21.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:21 smithi086 ceph-mon[88839]: failed to encode map e126 with expected crc 2024-04-02T05:16:21.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:21 smithi086 ceph-mon[88839]: failed to encode map e126 with expected crc 2024-04-02T05:16:21.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:21 smithi086 ceph-mon[88839]: failed to encode map e126 with expected crc 2024-04-02T05:16:21.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:21 smithi086 ceph-mon[88839]: failed to encode map e127 with expected crc 2024-04-02T05:16:21.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:21 smithi086 ceph-mon[88839]: failed to encode map e127 with expected crc 2024-04-02T05:16:21.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:21 smithi086 ceph-mon[88839]: failed to encode map e127 with expected crc 2024-04-02T05:16:21.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:21 smithi086 ceph-mon[88839]: failed to encode map e127 with expected crc 2024-04-02T05:16:21.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:21 smithi086 ceph-mon[88839]: osdmap e127: 8 total, 8 up, 8 in 2024-04-02T05:16:21.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:21 smithi086 ceph-mon[88839]: failed to encode map e127 with expected crc 2024-04-02T05:16:21.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:21 smithi086 ceph-mon[88839]: failed to encode map e127 with expected crc 2024-04-02T05:16:21.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:21 smithi086 ceph-mon[88839]: failed to encode map e127 with expected crc 2024-04-02T05:16:21.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:21 smithi086 ceph-mon[88839]: failed to encode map e127 with expected crc 2024-04-02T05:16:21.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:21 smithi086 ceph-mon[86443]: failed to encode map e126 with expected crc 2024-04-02T05:16:21.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:21 smithi086 ceph-mon[86443]: failed to encode map e126 with expected crc 2024-04-02T05:16:21.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:21 smithi086 ceph-mon[86443]: failed to encode map e126 with expected crc 2024-04-02T05:16:21.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:21 smithi086 ceph-mon[86443]: failed to encode map e126 with expected crc 2024-04-02T05:16:21.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:21 smithi086 ceph-mon[86443]: failed to encode map e126 with expected crc 2024-04-02T05:16:21.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:21 smithi086 ceph-mon[86443]: failed to encode map e126 with expected crc 2024-04-02T05:16:21.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:21 smithi086 ceph-mon[86443]: failed to encode map e127 with expected crc 2024-04-02T05:16:21.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:21 smithi086 ceph-mon[86443]: failed to encode map e127 with expected crc 2024-04-02T05:16:21.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:21 smithi086 ceph-mon[86443]: failed to encode map e127 with expected crc 2024-04-02T05:16:21.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:21 smithi086 ceph-mon[86443]: failed to encode map e127 with expected crc 2024-04-02T05:16:21.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:21 smithi086 ceph-mon[86443]: osdmap e127: 8 total, 8 up, 8 in 2024-04-02T05:16:21.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:21 smithi086 ceph-mon[86443]: failed to encode map e127 with expected crc 2024-04-02T05:16:21.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:21 smithi086 ceph-mon[86443]: failed to encode map e127 with expected crc 2024-04-02T05:16:21.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:21 smithi086 ceph-mon[86443]: failed to encode map e127 with expected crc 2024-04-02T05:16:21.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:21 smithi086 ceph-mon[86443]: failed to encode map e127 with expected crc 2024-04-02T05:16:22.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:22 smithi086 ceph-mon[86443]: failed to encode map e127 with expected crc 2024-04-02T05:16:22.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:22 smithi086 ceph-mon[86443]: failed to encode map e127 with expected crc 2024-04-02T05:16:22.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:22 smithi086 ceph-mon[86443]: failed to encode map e127 with expected crc 2024-04-02T05:16:22.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:22 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:16:22.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:22 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:22.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:22 smithi086 ceph-mon[86443]: pgmap v85: 161 pgs: 1 active+clean+remapped, 9 active+undersized+degraded, 12 active+undersized, 139 active+clean; 585 KiB data, 138 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 41/723 objects degraded (5.671%); 7/723 objects misplaced (0.968%); 22 B/s, 1 objects/s recovering 2024-04-02T05:16:22.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:22 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:22.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:22 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:16:22.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:22 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:16:22.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:22 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:22.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:22 smithi086 ceph-mon[88839]: failed to encode map e127 with expected crc 2024-04-02T05:16:22.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:22 smithi086 ceph-mon[88839]: failed to encode map e127 with expected crc 2024-04-02T05:16:22.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:22 smithi086 ceph-mon[88839]: failed to encode map e127 with expected crc 2024-04-02T05:16:22.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:22 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:16:22.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:22 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:22.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:22 smithi086 ceph-mon[88839]: pgmap v85: 161 pgs: 1 active+clean+remapped, 9 active+undersized+degraded, 12 active+undersized, 139 active+clean; 585 KiB data, 138 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 41/723 objects degraded (5.671%); 7/723 objects misplaced (0.968%); 22 B/s, 1 objects/s recovering 2024-04-02T05:16:22.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:22 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:22.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:22 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:16:22.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:22 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:16:22.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:22 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:22.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:22 smithi192 ceph-mon[68000]: failed to encode map e127 with expected crc 2024-04-02T05:16:22.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:22 smithi192 ceph-mon[68000]: failed to encode map e127 with expected crc 2024-04-02T05:16:22.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:22 smithi192 ceph-mon[68000]: failed to encode map e127 with expected crc 2024-04-02T05:16:22.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:22 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:16:22.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:22 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:22.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:22 smithi192 ceph-mon[68000]: pgmap v85: 161 pgs: 1 active+clean+remapped, 9 active+undersized+degraded, 12 active+undersized, 139 active+clean; 585 KiB data, 138 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 41/723 objects degraded (5.671%); 7/723 objects misplaced (0.968%); 22 B/s, 1 objects/s recovering 2024-04-02T05:16:22.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:22 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:22.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:22 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:16:22.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:22 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:16:22.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:22 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:23.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:16:23.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:23.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:23.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all nvmeof 2024-04-02T05:16:23.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:23.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:23.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:23.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:23.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:23.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:23.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:23.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: Upgrade: Finalizing container_image settings 2024-04-02T05:16:23.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:16:23.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:16:23.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-02T05:16:23.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-02T05:16:23.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:16:23.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:16:23.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-02T05:16:23.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:16:23.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:16:23.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:16:23.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:16:23.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:16:23.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:16:23.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:16:23.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:16:23.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:16:23.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:16:23.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:16:23.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:16:23.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:16:23.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:16:23.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:16:23.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:16:23.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-02T05:16:23.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.681 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.681 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.681 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.681 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.681 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.681 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.681 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.681 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.681 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.681 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.681 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.681 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: Upgrade: Complete! 2024-04-02T05:16:23.681 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:16:23.681 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:16:23.682 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-02T05:16:23.682 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:23.682 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:23.682 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:16:23.682 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:16:23.682 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:23.682 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:16:23.682 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:16:23.682 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:23.682 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:23 smithi192 ceph-mon[68000]: Health check update: Degraded data redundancy: 41/723 objects degraded (5.671%), 9 pgs degraded (PG_DEGRADED) 2024-04-02T05:16:23.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:16:23] "GET /metrics HTTP/1.1" 200 38153 "" "Prometheus/2.43.0" 2024-04-02T05:16:23.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:16:23.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:23.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:23.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all nvmeof 2024-04-02T05:16:23.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:23.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:23.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:23.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:23.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:23.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:23.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:23.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: Upgrade: Finalizing container_image settings 2024-04-02T05:16:23.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:16:23.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:16:23.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-02T05:16:23.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-02T05:16:23.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:16:23.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:16:23.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-02T05:16:23.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:16:23.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:16:23.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:16:23.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:16:23.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:16:23.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:16:23.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:16:23.836 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:23.836 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:23.836 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all nvmeof 2024-04-02T05:16:23.836 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:23.836 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:23.836 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:23.836 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:23.836 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:23.836 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:23.836 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:23.836 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: Upgrade: Finalizing container_image settings 2024-04-02T05:16:23.836 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:16:23.836 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:16:23.836 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-02T05:16:23.836 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.836 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.837 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-02T05:16:23.837 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:16:23.837 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:16:23.837 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-02T05:16:23.837 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:16:23.837 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:16:23.837 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:16:23.837 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:16:23.837 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:16:23.837 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:16:23.837 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:16:23.838 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:16:23.838 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.838 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.838 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:16:23.838 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:16:23.838 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:16:23.838 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:16:23.838 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:16:23.839 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:16:23.839 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:16:23.839 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:16:23.839 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-02T05:16:23.839 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.839 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.839 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.839 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.839 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.839 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.839 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.840 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.840 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.840 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.840 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:16:23.840 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:16:23.840 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.840 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.840 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:16:23.840 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:16:23.840 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:16:23.840 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:16:23.840 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:16:23.840 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:16:23.841 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:16:23.841 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:16:23.841 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-02T05:16:23.841 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.841 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.841 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.841 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.841 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.841 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.841 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.841 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.841 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.841 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.842 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.842 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.842 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: Upgrade: Complete! 2024-04-02T05:16:23.842 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:16:23.842 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:16:23.842 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-02T05:16:23.842 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:23.842 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:23.842 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:16:23.842 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:16:23.842 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:23.842 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:16:23.842 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:16:23.842 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:23.843 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[88839]: Health check update: Degraded data redundancy: 41/723 objects degraded (5.671%), 9 pgs degraded (PG_DEGRADED) 2024-04-02T05:16:23.843 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.843 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:16:23.843 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: Upgrade: Complete! 2024-04-02T05:16:23.843 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:16:23.843 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:16:23.843 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-02T05:16:23.843 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:23.843 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:23.843 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:16:23.843 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:16:23.843 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:23.844 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:16:23.844 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:16:23.844 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:23.844 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:23 smithi086 ceph-mon[86443]: Health check update: Degraded data redundancy: 41/723 objects degraded (5.671%), 9 pgs degraded (PG_DEGRADED) 2024-04-02T05:16:24.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:24 smithi192 ceph-mon[68000]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 54 B/s, 2 objects/s recovering 2024-04-02T05:16:24.734 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:24 smithi086 ceph-mon[88839]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 54 B/s, 2 objects/s recovering 2024-04-02T05:16:24.735 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:24 smithi086 ceph-mon[86443]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 54 B/s, 2 objects/s recovering 2024-04-02T05:16:25.648 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:25 smithi086 ceph-mon[88839]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 41/723 objects degraded (5.671%), 9 pgs degraded) 2024-04-02T05:16:25.649 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:25 smithi086 ceph-mon[88839]: Cluster is now healthy 2024-04-02T05:16:25.649 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:25 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:25.649 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:25 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:16:25.649 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:25 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:25.649 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:25 smithi086 ceph-mon[86443]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 41/723 objects degraded (5.671%), 9 pgs degraded) 2024-04-02T05:16:25.649 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:25 smithi086 ceph-mon[86443]: Cluster is now healthy 2024-04-02T05:16:25.649 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:25 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:25.649 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:25 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:16:25.649 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:25 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:25.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:25 smithi192 ceph-mon[68000]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 41/723 objects degraded (5.671%), 9 pgs degraded) 2024-04-02T05:16:25.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:25 smithi192 ceph-mon[68000]: Cluster is now healthy 2024-04-02T05:16:25.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:25 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:25.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:25 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:16:25.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:25 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:26.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:26 smithi192 ceph-mon[68000]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s; 44 B/s, 2 objects/s recovering 2024-04-02T05:16:26.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:26 smithi086 ceph-mon[86443]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s; 44 B/s, 2 objects/s recovering 2024-04-02T05:16:26.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:26 smithi086 ceph-mon[88839]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s; 44 B/s, 2 objects/s recovering 2024-04-02T05:16:29.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:29 smithi086 ceph-mon[86443]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 39 B/s, 1 objects/s recovering 2024-04-02T05:16:29.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:29 smithi086 ceph-mon[88839]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 39 B/s, 1 objects/s recovering 2024-04-02T05:16:29.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:29 smithi192 ceph-mon[68000]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 39 B/s, 1 objects/s recovering 2024-04-02T05:16:31.058 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch ps' 2024-04-02T05:16:31.603 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:31 smithi086 ceph-mon[86443]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s; 32 B/s, 1 objects/s recovering 2024-04-02T05:16:31.604 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:31 smithi086 ceph-mon[86443]: from='client.64228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:31.604 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:31 smithi086 ceph-mon[88839]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s; 32 B/s, 1 objects/s recovering 2024-04-02T05:16:31.604 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:31 smithi086 ceph-mon[88839]: from='client.64228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:31.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:31 smithi192 ceph-mon[68000]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s; 32 B/s, 1 objects/s recovering 2024-04-02T05:16:31.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:31 smithi192 ceph-mon[68000]: from='client.64228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:33.013 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:32 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:16:33.013 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:32 smithi086 ceph-mon[86443]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s; 28 B/s, 1 objects/s recovering 2024-04-02T05:16:33.013 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:32 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:16:33.013 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:32 smithi086 ceph-mon[88839]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s; 28 B/s, 1 objects/s recovering 2024-04-02T05:16:33.014 INFO:teuthology.orchestra.run.smithi086.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T05:16:33.014 INFO:teuthology.orchestra.run.smithi086.stdout:alertmanager.a smithi086 *:9093,9094 running (4m) 15s ago 26m 26.3M - 0.25.0 c8568f914cd2 660ea62bb6b9 2024-04-02T05:16:33.014 INFO:teuthology.orchestra.run.smithi086.stdout:grafana.a smithi192 *:3000 running (4m) 2m ago 26m 79.2M - 9.4.12 2bacad6d85d8 259af9e63aa5 2024-04-02T05:16:33.014 INFO:teuthology.orchestra.run.smithi086.stdout:iscsi.foo.smithi086.rsbqwx smithi086 running (4m) 15s ago 25m 58.2M - 3.5 e1d6a67b021e fe728eb940f7 2024-04-02T05:16:33.014 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.x smithi192 *:8443,9283,8765 running (5m) 2m ago 30m 449M - 19.0.0-1979-g6cec793d 78f830190839 dd1e6d0f0a95 2024-04-02T05:16:33.014 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.y smithi086 *:8443,9283,8765 running (15m) 15s ago 32m 504M - 19.0.0-1979-g6cec793d 78f830190839 3dca54a018ab 2024-04-02T05:16:33.014 INFO:teuthology.orchestra.run.smithi086.stdout:mon.a smithi086 running (2m) 15s ago 32m 47.2M 2048M 19.0.0-1979-g6cec793d 78f830190839 8e478dd4da2d 2024-04-02T05:16:33.014 INFO:teuthology.orchestra.run.smithi086.stdout:mon.b smithi192 running (3m) 2m ago 31m 36.2M 2048M 19.0.0-1979-g6cec793d 78f830190839 4125e997188f 2024-04-02T05:16:33.014 INFO:teuthology.orchestra.run.smithi086.stdout:mon.c smithi086 running (2m) 15s ago 31m 39.7M 2048M 19.0.0-1979-g6cec793d 78f830190839 8180999b3142 2024-04-02T05:16:33.014 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.a smithi086 *:9100 running (5m) 15s ago 27m 15.9M - 1.5.0 0da6a335fe13 05a844569d7b 2024-04-02T05:16:33.014 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.b smithi192 *:9100 running (5m) 2m ago 27m 17.4M - 1.5.0 0da6a335fe13 1a7d6788d58b 2024-04-02T05:16:33.014 INFO:teuthology.orchestra.run.smithi086.stdout:osd.0 smithi086 running (55s) 15s ago 30m 43.2M 2992M 19.0.0-1979-g6cec793d 78f830190839 e521a0ef7daf 2024-04-02T05:16:33.015 INFO:teuthology.orchestra.run.smithi086.stdout:osd.1 smithi086 running (19s) 15s ago 30m 12.5M 2992M 19.0.0-1979-g6cec793d 78f830190839 76fe72844771 2024-04-02T05:16:33.015 INFO:teuthology.orchestra.run.smithi086.stdout:osd.2 smithi086 running (29m) 15s ago 29m 55.1M 2992M 17.2.0 e1d6a67b021e 472c860a9af8 2024-04-02T05:16:33.015 INFO:teuthology.orchestra.run.smithi086.stdout:osd.3 smithi086 running (29m) 15s ago 29m 54.9M 2992M 17.2.0 e1d6a67b021e 327538dac44e 2024-04-02T05:16:33.015 INFO:teuthology.orchestra.run.smithi086.stdout:osd.4 smithi192 running (29m) 2m ago 29m 56.1M 3248M 17.2.0 e1d6a67b021e 6535d7919a2d 2024-04-02T05:16:33.015 INFO:teuthology.orchestra.run.smithi086.stdout:osd.5 smithi192 running (28m) 2m ago 28m 54.9M 3248M 17.2.0 e1d6a67b021e 7259249605f2 2024-04-02T05:16:33.015 INFO:teuthology.orchestra.run.smithi086.stdout:osd.6 smithi192 running (28m) 2m ago 28m 52.9M 3248M 17.2.0 e1d6a67b021e bb7d171b02a1 2024-04-02T05:16:33.015 INFO:teuthology.orchestra.run.smithi086.stdout:osd.7 smithi192 running (27m) 2m ago 27m 54.9M 3248M 17.2.0 e1d6a67b021e 95f5a0353999 2024-04-02T05:16:33.015 INFO:teuthology.orchestra.run.smithi086.stdout:prometheus.a smithi192 *:9095 running (4m) 2m ago 27m 65.1M - 2.43.0 a07b618ecd1d 8378fcdb60e3 2024-04-02T05:16:33.015 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi086.bxzxlp smithi086 *:8000 running (25m) 15s ago 25m 79.5M - 17.2.0 e1d6a67b021e 6725785fbc53 2024-04-02T05:16:33.015 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi192.pmlbot smithi192 *:8000 running (25m) 2m ago 25m 77.9M - 17.2.0 e1d6a67b021e 5ae3638f8b1c 2024-04-02T05:16:33.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:32 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:16:33.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:32 smithi192 ceph-mon[68000]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s; 28 B/s, 1 objects/s recovering 2024-04-02T05:16:33.747 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:16:33 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:16:33] "GET /metrics HTTP/1.1" 200 38181 "" "Prometheus/2.43.0" 2024-04-02T05:16:33.776 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 2'"'"'' 2024-04-02T05:16:34.082 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:33 smithi086 ceph-mon[86443]: from='client.54291 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:34.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:33 smithi086 ceph-mon[88839]: from='client.54291 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:34.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:33 smithi192 ceph-mon[68000]: from='client.54291 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:34.979 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:34 smithi086 ceph-mon[88839]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 18 B/s, 0 objects/s recovering 2024-04-02T05:16:34.979 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:34 smithi086 ceph-mon[86443]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 18 B/s, 0 objects/s recovering 2024-04-02T05:16:35.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:34 smithi192 ceph-mon[68000]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 18 B/s, 0 objects/s recovering 2024-04-02T05:16:35.811 INFO:teuthology.orchestra.run.smithi086.stdout:true 2024-04-02T05:16:36.086 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:35 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/736594834' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:36.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:35 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/736594834' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:36.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:35 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/736594834' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:36.560 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 7'"'"'' 2024-04-02T05:16:37.117 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:36 smithi086 ceph-mon[88839]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:16:37.118 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:36 smithi086 ceph-mon[86443]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:16:37.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:36 smithi192 ceph-mon[68000]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:16:39.669 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:39 smithi086 ceph-mon[86443]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:16:39.669 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:39 smithi086 ceph-mon[88839]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:16:39.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:39 smithi192 ceph-mon[68000]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:16:40.031 INFO:teuthology.orchestra.run.smithi086.stdout:true 2024-04-02T05:16:40.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:40 smithi192 ceph-mon[68000]: from='client.64246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:40.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:40 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:16:40.722 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:40 smithi086 ceph-mon[88839]: from='client.64246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:40.722 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:40 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:16:40.723 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:40 smithi086 ceph-mon[86443]: from='client.64246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:40.723 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:40 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:16:40.747 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch upgrade status' 2024-04-02T05:16:41.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:41 smithi086 ceph-mon[86443]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:16:41.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:41 smithi086 ceph-mon[88839]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:16:41.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:41 smithi192 ceph-mon[68000]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:16:42.671 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:16:42.671 INFO:teuthology.orchestra.run.smithi086.stdout: "target_image": null, 2024-04-02T05:16:42.671 INFO:teuthology.orchestra.run.smithi086.stdout: "in_progress": false, 2024-04-02T05:16:42.672 INFO:teuthology.orchestra.run.smithi086.stdout: "which": "", 2024-04-02T05:16:42.672 INFO:teuthology.orchestra.run.smithi086.stdout: "services_complete": [], 2024-04-02T05:16:42.672 INFO:teuthology.orchestra.run.smithi086.stdout: "progress": null, 2024-04-02T05:16:42.672 INFO:teuthology.orchestra.run.smithi086.stdout: "message": "", 2024-04-02T05:16:42.672 INFO:teuthology.orchestra.run.smithi086.stdout: "is_paused": false 2024-04-02T05:16:42.672 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:16:43.326 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph health detail' 2024-04-02T05:16:43.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:16:43 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:16:43] "GET /metrics HTTP/1.1" 200 38177 "" "Prometheus/2.43.0" 2024-04-02T05:16:43.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:43 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:16:43.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:43 smithi086 ceph-mon[86443]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:16:43.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:43 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:16:43.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:43 smithi086 ceph-mon[88839]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:16:43.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:43 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:16:43.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:43 smithi192 ceph-mon[68000]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:16:44.625 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:44 smithi086 ceph-mon[88839]: from='client.54309 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:44.626 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:44 smithi086 ceph-mon[86443]: from='client.54309 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:44.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:44 smithi192 ceph-mon[68000]: from='client.54309 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:45.389 INFO:teuthology.orchestra.run.smithi086.stdout:HEALTH_OK 2024-04-02T05:16:45.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:45 smithi192 ceph-mon[68000]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:16:45.757 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:45 smithi086 ceph-mon[86443]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:16:45.757 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:45 smithi086 ceph-mon[88839]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:16:46.073 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1' 2024-04-02T05:16:46.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:46 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/2582983686' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T05:16:46.694 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:46 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/2582983686' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T05:16:46.694 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:46 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/2582983686' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T05:16:47.670 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:47 smithi086 ceph-mon[88839]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:16:47.671 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:47 smithi086 ceph-mon[86443]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:16:47.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:47 smithi192 ceph-mon[68000]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:16:49.441 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:49 smithi086 ceph-mon[88839]: from='client.54321 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:49.442 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:49 smithi086 ceph-mon[88839]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:16:49.442 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:49 smithi086 ceph-mon[86443]: from='client.54321 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:49.442 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:49 smithi086 ceph-mon[86443]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:16:49.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:49 smithi192 ceph-mon[68000]: from='client.54321 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:49.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:49 smithi192 ceph-mon[68000]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:16:49.860 INFO:teuthology.orchestra.run.smithi086.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:16:50.600 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-02T05:16:51.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:50 smithi192 ceph-mon[68000]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:16:51.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:50 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:51.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:50 smithi192 ceph-mon[68000]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:16:51.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:50 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:16:51.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:50 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:16:51.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:50 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:51.190 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:50 smithi086 ceph-mon[88839]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:16:51.190 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:50 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:51.190 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:50 smithi086 ceph-mon[88839]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:16:51.190 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:50 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:16:51.190 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:50 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:16:51.190 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:50 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:51.191 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:50 smithi086 ceph-mon[86443]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:16:51.191 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:50 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:51.191 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:50 smithi086 ceph-mon[86443]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:16:51.191 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:50 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:16:51.191 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:50 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:16:51.191 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:50 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:52.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:51 smithi192 ceph-mon[68000]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:16:52.197 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:51 smithi086 ceph-mon[86443]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:16:52.198 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:51 smithi086 ceph-mon[88839]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:16:52.817 INFO:teuthology.orchestra.run.smithi086.stdout:true 2024-04-02T05:16:53.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:52 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:16:53.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:52 smithi086 ceph-mon[88839]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:16:53.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:52 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:16:53.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:52 smithi086 ceph-mon[86443]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:16:53.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:52 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:16:53.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:52 smithi192 ceph-mon[68000]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:16:53.481 INFO:teuthology.orchestra.run.smithi086.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T05:16:53.481 INFO:teuthology.orchestra.run.smithi086.stdout:alertmanager.a smithi086 *:9093,9094 running (4m) 35s ago 26m 26.3M - 0.25.0 c8568f914cd2 660ea62bb6b9 2024-04-02T05:16:53.481 INFO:teuthology.orchestra.run.smithi086.stdout:grafana.a smithi192 *:3000 running (4m) 2m ago 26m 79.2M - 9.4.12 2bacad6d85d8 259af9e63aa5 2024-04-02T05:16:53.481 INFO:teuthology.orchestra.run.smithi086.stdout:iscsi.foo.smithi086.rsbqwx smithi086 running (4m) 35s ago 25m 58.2M - 3.5 e1d6a67b021e fe728eb940f7 2024-04-02T05:16:53.481 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.x smithi192 *:8443,9283,8765 running (6m) 2m ago 31m 449M - 19.0.0-1979-g6cec793d 78f830190839 dd1e6d0f0a95 2024-04-02T05:16:53.481 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.y smithi086 *:8443,9283,8765 running (15m) 35s ago 32m 504M - 19.0.0-1979-g6cec793d 78f830190839 3dca54a018ab 2024-04-02T05:16:53.481 INFO:teuthology.orchestra.run.smithi086.stdout:mon.a smithi086 running (2m) 35s ago 33m 47.2M 2048M 19.0.0-1979-g6cec793d 78f830190839 8e478dd4da2d 2024-04-02T05:16:53.481 INFO:teuthology.orchestra.run.smithi086.stdout:mon.b smithi192 running (3m) 2m ago 31m 36.2M 2048M 19.0.0-1979-g6cec793d 78f830190839 4125e997188f 2024-04-02T05:16:53.481 INFO:teuthology.orchestra.run.smithi086.stdout:mon.c smithi086 running (2m) 35s ago 31m 39.7M 2048M 19.0.0-1979-g6cec793d 78f830190839 8180999b3142 2024-04-02T05:16:53.481 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.a smithi086 *:9100 running (5m) 35s ago 27m 15.9M - 1.5.0 0da6a335fe13 05a844569d7b 2024-04-02T05:16:53.481 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.b smithi192 *:9100 running (5m) 2m ago 27m 17.4M - 1.5.0 0da6a335fe13 1a7d6788d58b 2024-04-02T05:16:53.481 INFO:teuthology.orchestra.run.smithi086.stdout:osd.0 smithi086 running (75s) 35s ago 30m 43.2M 2992M 19.0.0-1979-g6cec793d 78f830190839 e521a0ef7daf 2024-04-02T05:16:53.481 INFO:teuthology.orchestra.run.smithi086.stdout:osd.1 smithi086 running (40s) 35s ago 30m 12.5M 2992M 19.0.0-1979-g6cec793d 78f830190839 76fe72844771 2024-04-02T05:16:53.482 INFO:teuthology.orchestra.run.smithi086.stdout:osd.2 smithi086 running (30m) 35s ago 30m 55.1M 2992M 17.2.0 e1d6a67b021e 472c860a9af8 2024-04-02T05:16:53.482 INFO:teuthology.orchestra.run.smithi086.stdout:osd.3 smithi086 running (29m) 35s ago 29m 54.9M 2992M 17.2.0 e1d6a67b021e 327538dac44e 2024-04-02T05:16:53.482 INFO:teuthology.orchestra.run.smithi086.stdout:osd.4 smithi192 running (29m) 2m ago 29m 56.1M 3248M 17.2.0 e1d6a67b021e 6535d7919a2d 2024-04-02T05:16:53.482 INFO:teuthology.orchestra.run.smithi086.stdout:osd.5 smithi192 running (29m) 2m ago 29m 54.9M 3248M 17.2.0 e1d6a67b021e 7259249605f2 2024-04-02T05:16:53.482 INFO:teuthology.orchestra.run.smithi086.stdout:osd.6 smithi192 running (28m) 2m ago 28m 52.9M 3248M 17.2.0 e1d6a67b021e bb7d171b02a1 2024-04-02T05:16:53.482 INFO:teuthology.orchestra.run.smithi086.stdout:osd.7 smithi192 running (28m) 2m ago 28m 54.9M 3248M 17.2.0 e1d6a67b021e 95f5a0353999 2024-04-02T05:16:53.482 INFO:teuthology.orchestra.run.smithi086.stdout:prometheus.a smithi192 *:9095 running (5m) 2m ago 27m 65.1M - 2.43.0 a07b618ecd1d 8378fcdb60e3 2024-04-02T05:16:53.482 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi086.bxzxlp smithi086 *:8000 running (26m) 35s ago 26m 79.5M - 17.2.0 e1d6a67b021e 6725785fbc53 2024-04-02T05:16:53.482 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi192.pmlbot smithi192 *:8000 running (26m) 2m ago 26m 77.9M - 17.2.0 e1d6a67b021e 5ae3638f8b1c 2024-04-02T05:16:53.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:16:53 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:16:53] "GET /metrics HTTP/1.1" 200 38177 "" "Prometheus/2.43.0" 2024-04-02T05:16:53.916 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:16:53.916 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": { 2024-04-02T05:16:53.916 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 3 2024-04-02T05:16:53.917 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:16:53.917 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": { 2024-04-02T05:16:53.917 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 2 2024-04-02T05:16:53.917 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:16:53.917 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": { 2024-04-02T05:16:53.917 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-04-02T05:16:53.917 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 2 2024-04-02T05:16:53.917 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:16:53.917 INFO:teuthology.orchestra.run.smithi086.stdout: "rgw": { 2024-04-02T05:16:53.917 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-02T05:16:53.917 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:16:53.917 INFO:teuthology.orchestra.run.smithi086.stdout: "overall": { 2024-04-02T05:16:53.917 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-04-02T05:16:53.917 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 7 2024-04-02T05:16:53.917 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-04-02T05:16:53.918 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:16:54.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:53 smithi192 ceph-mon[68000]: from='client.64270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:54.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:53 smithi192 ceph-mon[68000]: from='client.54333 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:54.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:53 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:54.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:53 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:16:54.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:53 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:54.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:53 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:54.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:53 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:54.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:53 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:54.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:53 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:54.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:53 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:54.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:53 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-02T05:16:54.253 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:53 smithi086 ceph-mon[86443]: from='client.64270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:54.253 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:53 smithi086 ceph-mon[86443]: from='client.54333 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:54.253 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:53 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:54.253 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:53 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:16:54.253 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:53 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:54.253 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:53 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:54.253 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:53 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:54.253 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:53 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:54.253 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:53 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:54.253 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:53 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:54.253 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:53 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-02T05:16:54.254 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:53 smithi086 ceph-mon[88839]: from='client.64270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:54.254 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:53 smithi086 ceph-mon[88839]: from='client.54333 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:54.254 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:53 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:54.254 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:53 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:16:54.254 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:53 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:54.254 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:53 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:54.254 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:53 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:54.254 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:53 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:54.254 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:53 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:54.254 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:53 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:54.254 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:53 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-02T05:16:54.266 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:16:54.266 INFO:teuthology.orchestra.run.smithi086.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", 2024-04-02T05:16:54.266 INFO:teuthology.orchestra.run.smithi086.stdout: "in_progress": true, 2024-04-02T05:16:54.266 INFO:teuthology.orchestra.run.smithi086.stdout: "which": "Upgrading daemons of type(s) crash,osd. Upgrade limited to 1 daemons (1 remaining).", 2024-04-02T05:16:54.266 INFO:teuthology.orchestra.run.smithi086.stdout: "services_complete": [], 2024-04-02T05:16:54.266 INFO:teuthology.orchestra.run.smithi086.stdout: "progress": "2/8 daemons upgraded", 2024-04-02T05:16:54.266 INFO:teuthology.orchestra.run.smithi086.stdout: "message": "Currently upgrading osd daemons", 2024-04-02T05:16:54.266 INFO:teuthology.orchestra.run.smithi086.stdout: "is_paused": false 2024-04-02T05:16:54.267 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:16:55.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:54 smithi086 ceph-mon[88839]: from='client.54339 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:55.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:54 smithi086 ceph-mon[88839]: Upgrade: Target is version 19.0.0-1979-g6cec793d (squid) 2024-04-02T05:16:55.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:54 smithi086 ceph-mon[88839]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7bd5b84926d4a0d6c51e5562e965bff211cec2740c9f4dec9b5e00a2bbb75eae'] 2024-04-02T05:16:55.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:54 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all mgr 2024-04-02T05:16:55.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:54 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all mon 2024-04-02T05:16:55.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:54 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all crash 2024-04-02T05:16:55.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:54 smithi086 ceph-mon[88839]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-02T05:16:55.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:54 smithi086 ceph-mon[88839]: Upgrade: osd.2 is safe to restart 2024-04-02T05:16:55.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:54 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/363020531' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:55.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:54 smithi086 ceph-mon[88839]: from='client.64294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:55.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:54 smithi086 ceph-mon[88839]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:16:55.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:54 smithi086 ceph-mon[86443]: from='client.54339 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:55.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:54 smithi086 ceph-mon[86443]: Upgrade: Target is version 19.0.0-1979-g6cec793d (squid) 2024-04-02T05:16:55.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:54 smithi086 ceph-mon[86443]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7bd5b84926d4a0d6c51e5562e965bff211cec2740c9f4dec9b5e00a2bbb75eae'] 2024-04-02T05:16:55.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:54 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all mgr 2024-04-02T05:16:55.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:54 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all mon 2024-04-02T05:16:55.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:54 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all crash 2024-04-02T05:16:55.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:54 smithi086 ceph-mon[86443]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-02T05:16:55.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:54 smithi086 ceph-mon[86443]: Upgrade: osd.2 is safe to restart 2024-04-02T05:16:55.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:54 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/363020531' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:55.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:54 smithi086 ceph-mon[86443]: from='client.64294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:55.085 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:54 smithi086 ceph-mon[86443]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:16:55.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:54 smithi192 ceph-mon[68000]: from='client.54339 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:55.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:54 smithi192 ceph-mon[68000]: Upgrade: Target is version 19.0.0-1979-g6cec793d (squid) 2024-04-02T05:16:55.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:54 smithi192 ceph-mon[68000]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7bd5b84926d4a0d6c51e5562e965bff211cec2740c9f4dec9b5e00a2bbb75eae'] 2024-04-02T05:16:55.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:54 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all mgr 2024-04-02T05:16:55.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:54 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all mon 2024-04-02T05:16:55.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:54 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all crash 2024-04-02T05:16:55.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:54 smithi192 ceph-mon[68000]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-02T05:16:55.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:54 smithi192 ceph-mon[68000]: Upgrade: osd.2 is safe to restart 2024-04-02T05:16:55.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:54 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/363020531' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:16:55.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:54 smithi192 ceph-mon[68000]: from='client.64294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:16:55.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:54 smithi192 ceph-mon[68000]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:16:56.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:55 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:16:56.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:55 smithi192 ceph-mon[68000]: Upgrade: Updating osd.2 2024-04-02T05:16:56.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:55 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:56.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:55 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-02T05:16:56.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:55 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:16:56.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:55 smithi192 ceph-mon[68000]: Deploying daemon osd.2 on smithi086 2024-04-02T05:16:56.199 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:55 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:16:56.199 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:55 smithi086 ceph-mon[88839]: Upgrade: Updating osd.2 2024-04-02T05:16:56.199 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:55 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:56.199 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:55 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-02T05:16:56.199 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:55 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:16:56.199 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:55 smithi086 ceph-mon[88839]: Deploying daemon osd.2 on smithi086 2024-04-02T05:16:56.200 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:55 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:16:56.200 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:55 smithi086 ceph-mon[86443]: Upgrade: Updating osd.2 2024-04-02T05:16:56.200 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:55 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:16:56.200 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:55 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-02T05:16:56.200 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:55 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:16:56.200 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:55 smithi086 ceph-mon[86443]: Deploying daemon osd.2 on smithi086 2024-04-02T05:16:57.170 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:56 smithi086 ceph-mon[88839]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:16:57.170 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:56 smithi086 ceph-mon[86443]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:16:57.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:56 smithi192 ceph-mon[68000]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:16:58.833 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:16:58 smithi086 systemd[1]: Stopping Ceph osd.2 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:16:58.833 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:16:58 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2[38252]: 2024-04-02T05:16:58.744+0000 7f7100c61700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T05:16:58.833 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:16:58 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2[38252]: 2024-04-02T05:16:58.744+0000 7f7100c61700 -1 osd.2 127 *** Got signal Terminated *** 2024-04-02T05:16:58.833 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:16:58 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2[38252]: 2024-04-02T05:16:58.744+0000 7f7100c61700 -1 osd.2 127 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T05:16:59.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:59 smithi192 ceph-mon[68000]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:16:59.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:16:59 smithi192 ceph-mon[68000]: osd.2 marked itself down and dead 2024-04-02T05:16:59.789 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:59 smithi086 ceph-mon[86443]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:16:59.789 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:16:59 smithi086 ceph-mon[86443]: osd.2 marked itself down and dead 2024-04-02T05:16:59.790 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:59 smithi086 ceph-mon[88839]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:16:59.790 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:16:59 smithi086 ceph-mon[88839]: osd.2 marked itself down and dead 2024-04-02T05:17:00.081 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:16:59 smithi086 podman[107153]: 2024-04-02 05:16:59.787197198 +0000 UTC m=+1.139204657 container died 472c860a9af84c15bf4fc5e8ced5d2a8a8f8ecf7b65af686aaf9c88e24f80c3c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, ceph=True, maintainer=Guillaume Abrioux , vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, GIT_BRANCH=HEAD, name=centos-stream, distribution-scope=public, io.openshift.expose-services=, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., architecture=x86_64) 2024-04-02T05:17:00.333 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:00 smithi086 podman[107153]: 2024-04-02 05:17:00.170030987 +0000 UTC m=+1.522038444 container cleanup 472c860a9af84c15bf4fc5e8ced5d2a8a8f8ecf7b65af686aaf9c88e24f80c3c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, release=754, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., architecture=x86_64, io.buildah.version=1.19.8, name=centos-stream, io.openshift.expose-services=, distribution-scope=public, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0) 2024-04-02T05:17:00.333 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:00 smithi086 bash[107153]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2 2024-04-02T05:17:00.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:00 smithi192 ceph-mon[68000]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-02T05:17:00.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:00 smithi192 ceph-mon[68000]: failed to encode map e128 with expected crc 2024-04-02T05:17:00.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:00 smithi192 ceph-mon[68000]: failed to encode map e128 with expected crc 2024-04-02T05:17:00.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:00 smithi192 ceph-mon[68000]: failed to encode map e128 with expected crc 2024-04-02T05:17:00.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:00 smithi192 ceph-mon[68000]: failed to encode map e128 with expected crc 2024-04-02T05:17:00.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:00 smithi192 ceph-mon[68000]: osdmap e128: 8 total, 7 up, 8 in 2024-04-02T05:17:00.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:00 smithi192 ceph-mon[68000]: failed to encode map e128 with expected crc 2024-04-02T05:17:00.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:00 smithi192 ceph-mon[68000]: failed to encode map e128 with expected crc 2024-04-02T05:17:00.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:00 smithi192 ceph-mon[68000]: osdmap e129: 8 total, 7 up, 8 in 2024-04-02T05:17:00.824 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:00 smithi086 podman[107170]: 2024-04-02 05:17:00.513078182 +0000 UTC m=+0.728394521 container remove 472c860a9af84c15bf4fc5e8ced5d2a8a8f8ecf7b65af686aaf9c88e24f80c3c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, GIT_CLEAN=True, io.buildah.version=1.19.8, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, architecture=x86_64, ceph=True, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-02T05:17:00.824 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:00 smithi086 ceph-mon[86443]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-02T05:17:00.824 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:00 smithi086 ceph-mon[86443]: failed to encode map e128 with expected crc 2024-04-02T05:17:00.824 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:00 smithi086 ceph-mon[86443]: failed to encode map e128 with expected crc 2024-04-02T05:17:00.825 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:00 smithi086 ceph-mon[86443]: failed to encode map e128 with expected crc 2024-04-02T05:17:00.825 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:00 smithi086 ceph-mon[86443]: failed to encode map e128 with expected crc 2024-04-02T05:17:00.825 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:00 smithi086 ceph-mon[86443]: osdmap e128: 8 total, 7 up, 8 in 2024-04-02T05:17:00.825 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:00 smithi086 ceph-mon[86443]: failed to encode map e128 with expected crc 2024-04-02T05:17:00.825 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:00 smithi086 ceph-mon[86443]: failed to encode map e128 with expected crc 2024-04-02T05:17:00.825 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:00 smithi086 ceph-mon[86443]: osdmap e129: 8 total, 7 up, 8 in 2024-04-02T05:17:00.825 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:00 smithi086 ceph-mon[88839]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-02T05:17:00.825 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:00 smithi086 ceph-mon[88839]: failed to encode map e128 with expected crc 2024-04-02T05:17:00.825 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:00 smithi086 ceph-mon[88839]: failed to encode map e128 with expected crc 2024-04-02T05:17:00.825 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:00 smithi086 ceph-mon[88839]: failed to encode map e128 with expected crc 2024-04-02T05:17:00.825 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:00 smithi086 ceph-mon[88839]: failed to encode map e128 with expected crc 2024-04-02T05:17:00.825 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:00 smithi086 ceph-mon[88839]: osdmap e128: 8 total, 7 up, 8 in 2024-04-02T05:17:00.826 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:00 smithi086 ceph-mon[88839]: failed to encode map e128 with expected crc 2024-04-02T05:17:00.826 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:00 smithi086 ceph-mon[88839]: failed to encode map e128 with expected crc 2024-04-02T05:17:00.826 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:00 smithi086 ceph-mon[88839]: osdmap e129: 8 total, 7 up, 8 in 2024-04-02T05:17:01.083 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:00 smithi086 podman[107260]: 2024-04-02 05:17:00.822106108 +0000 UTC m=+0.100404280 container create 54d5d11083c8ed4882b3619c6afdbce176e9c830b0b3bd5f6cf36b02e8327079 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2-deactivate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-6cec793, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-02T05:17:01.083 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:00 smithi086 podman[107260]: 2024-04-02 05:17:00.740177847 +0000 UTC m=+0.018476038 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:17:01.358 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:01 smithi086 podman[107260]: 2024-04-02 05:17:01.103018241 +0000 UTC m=+0.381316423 container init 54d5d11083c8ed4882b3619c6afdbce176e9c830b0b3bd5f6cf36b02e8327079 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-6cec793, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=) 2024-04-02T05:17:01.358 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:01 smithi086 podman[107260]: 2024-04-02 05:17:01.10652572 +0000 UTC m=+0.384823901 container start 54d5d11083c8ed4882b3619c6afdbce176e9c830b0b3bd5f6cf36b02e8327079 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2-deactivate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-6cec793, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-04-02T05:17:01.358 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:01 smithi086 podman[107260]: 2024-04-02 05:17:01.199951483 +0000 UTC m=+0.478249674 container attach 54d5d11083c8ed4882b3619c6afdbce176e9c830b0b3bd5f6cf36b02e8327079 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, ceph=True, RELEASE=squid-6cec793, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-04-02T05:17:01.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:01 smithi192 ceph-mon[68000]: pgmap v105: 161 pgs: 14 stale+active+clean, 147 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:17:01.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:01 smithi192 ceph-mon[68000]: failed to encode map e129 with expected crc 2024-04-02T05:17:01.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:01 smithi192 ceph-mon[68000]: failed to encode map e129 with expected crc 2024-04-02T05:17:01.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:01 smithi192 ceph-mon[68000]: failed to encode map e129 with expected crc 2024-04-02T05:17:01.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:01 smithi192 ceph-mon[68000]: failed to encode map e129 with expected crc 2024-04-02T05:17:01.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:01 smithi192 ceph-mon[68000]: failed to encode map e129 with expected crc 2024-04-02T05:17:01.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:01 smithi192 ceph-mon[68000]: failed to encode map e129 with expected crc 2024-04-02T05:17:01.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:01 smithi192 ceph-mon[68000]: failed to encode map e129 with expected crc 2024-04-02T05:17:01.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:01 smithi192 ceph-mon[68000]: failed to encode map e129 with expected crc 2024-04-02T05:17:01.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:01 smithi192 ceph-mon[68000]: failed to encode map e129 with expected crc 2024-04-02T05:17:01.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:01 smithi192 ceph-mon[68000]: failed to encode map e129 with expected crc 2024-04-02T05:17:01.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:01 smithi192 ceph-mon[68000]: failed to encode map e129 with expected crc 2024-04-02T05:17:01.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:01 smithi192 ceph-mon[68000]: failed to encode map e129 with expected crc 2024-04-02T05:17:01.696 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:01 smithi086 podman[107260]: 2024-04-02 05:17:01.405290907 +0000 UTC m=+0.683589073 container died 54d5d11083c8ed4882b3619c6afdbce176e9c830b0b3bd5f6cf36b02e8327079 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, RELEASE=squid-6cec793, org.label-schema.schema-version=1.0) 2024-04-02T05:17:01.697 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:01 smithi086 ceph-mon[88839]: pgmap v105: 161 pgs: 14 stale+active+clean, 147 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:17:01.697 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:01 smithi086 ceph-mon[88839]: failed to encode map e129 with expected crc 2024-04-02T05:17:01.697 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:01 smithi086 ceph-mon[88839]: failed to encode map e129 with expected crc 2024-04-02T05:17:01.697 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:01 smithi086 ceph-mon[88839]: failed to encode map e129 with expected crc 2024-04-02T05:17:01.697 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:01 smithi086 ceph-mon[88839]: failed to encode map e129 with expected crc 2024-04-02T05:17:01.697 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:01 smithi086 ceph-mon[88839]: failed to encode map e129 with expected crc 2024-04-02T05:17:01.697 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:01 smithi086 ceph-mon[88839]: failed to encode map e129 with expected crc 2024-04-02T05:17:01.697 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:01 smithi086 ceph-mon[88839]: failed to encode map e129 with expected crc 2024-04-02T05:17:01.697 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:01 smithi086 ceph-mon[88839]: failed to encode map e129 with expected crc 2024-04-02T05:17:01.697 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:01 smithi086 ceph-mon[88839]: failed to encode map e129 with expected crc 2024-04-02T05:17:01.697 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:01 smithi086 ceph-mon[88839]: failed to encode map e129 with expected crc 2024-04-02T05:17:01.697 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:01 smithi086 ceph-mon[88839]: failed to encode map e129 with expected crc 2024-04-02T05:17:01.697 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:01 smithi086 ceph-mon[88839]: failed to encode map e129 with expected crc 2024-04-02T05:17:01.698 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:01 smithi086 ceph-mon[86443]: pgmap v105: 161 pgs: 14 stale+active+clean, 147 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:17:01.698 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:01 smithi086 ceph-mon[86443]: failed to encode map e129 with expected crc 2024-04-02T05:17:01.698 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:01 smithi086 ceph-mon[86443]: failed to encode map e129 with expected crc 2024-04-02T05:17:01.698 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:01 smithi086 ceph-mon[86443]: failed to encode map e129 with expected crc 2024-04-02T05:17:01.698 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:01 smithi086 ceph-mon[86443]: failed to encode map e129 with expected crc 2024-04-02T05:17:01.698 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:01 smithi086 ceph-mon[86443]: failed to encode map e129 with expected crc 2024-04-02T05:17:01.698 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:01 smithi086 ceph-mon[86443]: failed to encode map e129 with expected crc 2024-04-02T05:17:01.698 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:01 smithi086 ceph-mon[86443]: failed to encode map e129 with expected crc 2024-04-02T05:17:01.698 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:01 smithi086 ceph-mon[86443]: failed to encode map e129 with expected crc 2024-04-02T05:17:01.698 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:01 smithi086 ceph-mon[86443]: failed to encode map e129 with expected crc 2024-04-02T05:17:01.698 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:01 smithi086 ceph-mon[86443]: failed to encode map e129 with expected crc 2024-04-02T05:17:01.698 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:01 smithi086 ceph-mon[86443]: failed to encode map e129 with expected crc 2024-04-02T05:17:01.698 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:01 smithi086 ceph-mon[86443]: failed to encode map e129 with expected crc 2024-04-02T05:17:02.333 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:01 smithi086 podman[107286]: 2024-04-02 05:17:01.978783868 +0000 UTC m=+0.563317055 container remove 54d5d11083c8ed4882b3619c6afdbce176e9c830b0b3bd5f6cf36b02e8327079 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2-deactivate, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=squid-6cec793, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-02T05:17:02.333 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:02 smithi086 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.2.service: Deactivated successfully. 2024-04-02T05:17:02.333 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:02 smithi086 systemd[1]: Stopped Ceph osd.2 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:17:02.333 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:02 smithi086 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.2.service: Consumed 13.580s CPU time. 2024-04-02T05:17:02.333 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:02 smithi086 systemd[1]: Starting Ceph osd.2 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:17:02.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:02 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:17:02.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:02 smithi192 ceph-mon[68000]: pgmap v107: 161 pgs: 14 active+undersized+degraded, 3 stale+active+clean, 27 active+undersized, 117 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 58/723 objects degraded (8.022%) 2024-04-02T05:17:02.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:02 smithi192 ceph-mon[68000]: Health check failed: Degraded data redundancy: 58/723 objects degraded (8.022%), 14 pgs degraded (PG_DEGRADED) 2024-04-02T05:17:02.700 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:02 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:17:02.700 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:02 smithi086 ceph-mon[86443]: pgmap v107: 161 pgs: 14 active+undersized+degraded, 3 stale+active+clean, 27 active+undersized, 117 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 58/723 objects degraded (8.022%) 2024-04-02T05:17:02.701 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:02 smithi086 ceph-mon[86443]: Health check failed: Degraded data redundancy: 58/723 objects degraded (8.022%), 14 pgs degraded (PG_DEGRADED) 2024-04-02T05:17:02.701 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:02 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:17:02.701 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:02 smithi086 ceph-mon[88839]: pgmap v107: 161 pgs: 14 active+undersized+degraded, 3 stale+active+clean, 27 active+undersized, 117 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 58/723 objects degraded (8.022%) 2024-04-02T05:17:02.701 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:02 smithi086 ceph-mon[88839]: Health check failed: Degraded data redundancy: 58/723 objects degraded (8.022%), 14 pgs degraded (PG_DEGRADED) 2024-04-02T05:17:02.701 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:02 smithi086 podman[107392]: 2024-04-02 05:17:02.325045336 +0000 UTC m=+0.017835378 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:17:02.701 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:02 smithi086 podman[107392]: 2024-04-02 05:17:02.454635678 +0000 UTC m=+0.147425691 container create 164982fc05840db0a61180f5650ee1e146b54826f4bf30e4b2a44e7b8cb7e76d (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2-activate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=squid-6cec793, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-04-02T05:17:03.083 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:02 smithi086 podman[107392]: 2024-04-02 05:17:02.698766665 +0000 UTC m=+0.391556687 container init 164982fc05840db0a61180f5650ee1e146b54826f4bf30e4b2a44e7b8cb7e76d (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2-activate, org.label-schema.build-date=20240326, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-6cec793, org.label-schema.vendor=CentOS) 2024-04-02T05:17:03.083 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:02 smithi086 podman[107392]: 2024-04-02 05:17:02.702476143 +0000 UTC m=+0.395266163 container start 164982fc05840db0a61180f5650ee1e146b54826f4bf30e4b2a44e7b8cb7e76d (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2-activate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, RELEASE=squid-6cec793, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-02T05:17:03.083 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:02 smithi086 podman[107392]: 2024-04-02 05:17:02.790504628 +0000 UTC m=+0.483294651 container attach 164982fc05840db0a61180f5650ee1e146b54826f4bf30e4b2a44e7b8cb7e76d (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2-activate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=squid-6cec793, org.label-schema.build-date=20240326) 2024-04-02T05:17:03.403 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2-activate[107406]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-02T05:17:03.403 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2-activate[107406]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-02T05:17:03.403 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:03 smithi086 bash[107392]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-02T05:17:03.403 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:03 smithi086 bash[107392]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-02T05:17:03.403 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:03 smithi086 bash[107392]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-02T05:17:03.403 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:03 smithi086 bash[107392]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-02T05:17:03.403 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:03 smithi086 bash[107392]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-02T05:17:03.403 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:03 smithi086 bash[107392]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-02T05:17:03.403 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:03 smithi086 bash[107392]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-02T05:17:03.403 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2-activate[107406]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-02T05:17:03.404 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2-activate[107406]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-02T05:17:03.404 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2-activate[107406]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-02T05:17:03.404 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2-activate[107406]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-02T05:17:03.404 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2-activate[107406]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-02T05:17:03.404 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:03 smithi086 podman[107392]: 2024-04-02 05:17:03.169487594 +0000 UTC m=+0.862277622 container died 164982fc05840db0a61180f5650ee1e146b54826f4bf30e4b2a44e7b8cb7e76d (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2-activate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, RELEASE=squid-6cec793, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-04-02T05:17:03.787 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:17:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:17:03] "GET /metrics HTTP/1.1" 200 38176 "" "Prometheus/2.43.0" 2024-04-02T05:17:04.083 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:03 smithi086 podman[107542]: 2024-04-02 05:17:03.785925366 +0000 UTC m=+0.611516423 container remove 164982fc05840db0a61180f5650ee1e146b54826f4bf30e4b2a44e7b8cb7e76d (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2-activate, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=squid-6cec793, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-02T05:17:04.083 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:03 smithi086 podman[107606]: 2024-04-02 05:17:03.902700488 +0000 UTC m=+0.017984986 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:17:04.473 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:04 smithi086 podman[107606]: 2024-04-02 05:17:04.203758128 +0000 UTC m=+0.319042618 container create 28594f089a8f1f8a58edecd0c4cb76aada31a33a43e6c05d3ad84a6d7a852c6b (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-6cec793, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0) 2024-04-02T05:17:04.473 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:04 smithi086 podman[107606]: 2024-04-02 05:17:04.447932775 +0000 UTC m=+0.563217265 container init 28594f089a8f1f8a58edecd0c4cb76aada31a33a43e6c05d3ad84a6d7a852c6b (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2, ceph=True, maintainer=Guillaume Abrioux , RELEASE=squid-6cec793, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-04-02T05:17:04.473 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:04 smithi086 podman[107606]: 2024-04-02 05:17:04.451180169 +0000 UTC m=+0.566464661 container start 28594f089a8f1f8a58edecd0c4cb76aada31a33a43e6c05d3ad84a6d7a852c6b (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=squid-6cec793, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-02T05:17:04.833 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:04 smithi086 ceph-osd[107626]: -- 172.21.15.86:0/2710603794 <== mon.0 v2:172.21.15.86:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d361099680 con 0x55d361062c00 2024-04-02T05:17:04.833 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:04 smithi086 bash[107606]: 28594f089a8f1f8a58edecd0c4cb76aada31a33a43e6c05d3ad84a6d7a852c6b 2024-04-02T05:17:04.833 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:04 smithi086 systemd[1]: Started Ceph osd.2 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:17:05.333 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:04 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2[107622]: 2024-04-02T05:17:04.995+0000 7fe7dfac27c0 -1 Falling back to public interface 2024-04-02T05:17:05.615 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:05 smithi086 ceph-mon[86443]: pgmap v108: 161 pgs: 18 active+undersized+degraded, 32 active+undersized, 111 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 88/723 objects degraded (12.172%) 2024-04-02T05:17:05.615 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:05 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:05.615 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:05 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:05.615 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:05 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:05.615 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:05 smithi086 ceph-mon[88839]: pgmap v108: 161 pgs: 18 active+undersized+degraded, 32 active+undersized, 111 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 88/723 objects degraded (12.172%) 2024-04-02T05:17:05.615 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:05 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:05.615 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:05 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:05.615 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:05 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:05.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:05 smithi192 ceph-mon[68000]: pgmap v108: 161 pgs: 18 active+undersized+degraded, 32 active+undersized, 111 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 88/723 objects degraded (12.172%) 2024-04-02T05:17:05.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:05 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:05.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:05 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:05.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:05 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:07.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:07 smithi192 ceph-mon[68000]: pgmap v109: 161 pgs: 18 active+undersized+degraded, 32 active+undersized, 111 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 88/723 objects degraded (12.172%) 2024-04-02T05:17:07.723 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:07 smithi086 ceph-mon[88839]: pgmap v109: 161 pgs: 18 active+undersized+degraded, 32 active+undersized, 111 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 88/723 objects degraded (12.172%) 2024-04-02T05:17:07.723 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:07 smithi086 ceph-mon[86443]: pgmap v109: 161 pgs: 18 active+undersized+degraded, 32 active+undersized, 111 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 88/723 objects degraded (12.172%) 2024-04-02T05:17:08.082 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:07 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2[107622]: 2024-04-02T05:17:07.740+0000 7fe7dfac27c0 -1 osd.2 0 read_superblock omap replica is missing. 2024-04-02T05:17:08.083 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:07 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2[107622]: 2024-04-02T05:17:07.766+0000 7fe7dfac27c0 -1 osd.2 127 log_to_monitors true 2024-04-02T05:17:08.600 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:08 smithi086 ceph-mon[86443]: from='osd.2 [v2:172.21.15.86:6818/3113900521,v1:172.21.15.86:6819/3113900521]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-02T05:17:08.601 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:08 smithi086 ceph-mon[88839]: from='osd.2 [v2:172.21.15.86:6818/3113900521,v1:172.21.15.86:6819/3113900521]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-02T05:17:08.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:08 smithi192 ceph-mon[68000]: from='osd.2 [v2:172.21.15.86:6818/3113900521,v1:172.21.15.86:6819/3113900521]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-02T05:17:09.082 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:17:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2[107622]: 2024-04-02T05:17:08.818+0000 7fe7ccf12700 -1 osd.2 127 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T05:17:09.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:09 smithi192 ceph-mon[68000]: pgmap v110: 161 pgs: 18 active+undersized+degraded, 32 active+undersized, 111 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 88/723 objects degraded (12.172%) 2024-04-02T05:17:09.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:09 smithi192 ceph-mon[68000]: from='osd.2 [v2:172.21.15.86:6818/3113900521,v1:172.21.15.86:6819/3113900521]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-02T05:17:09.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:09 smithi192 ceph-mon[68000]: failed to encode map e130 with expected crc 2024-04-02T05:17:09.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:09 smithi192 ceph-mon[68000]: osdmap e130: 8 total, 7 up, 8 in 2024-04-02T05:17:09.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:09 smithi192 ceph-mon[68000]: from='osd.2 [v2:172.21.15.86:6818/3113900521,v1:172.21.15.86:6819/3113900521]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T05:17:09.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:09 smithi192 ceph-mon[68000]: failed to encode map e130 with expected crc 2024-04-02T05:17:09.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:09 smithi192 ceph-mon[68000]: failed to encode map e130 with expected crc 2024-04-02T05:17:09.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:09 smithi192 ceph-mon[68000]: from='osd.2 [v2:172.21.15.86:6818/3113900521,v1:172.21.15.86:6819/3113900521]' entity='osd.2' 2024-04-02T05:17:09.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:09.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:09.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:09 smithi192 ceph-mon[68000]: failed to encode map e130 with expected crc 2024-04-02T05:17:09.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:09 smithi086 ceph-mon[88839]: pgmap v110: 161 pgs: 18 active+undersized+degraded, 32 active+undersized, 111 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 88/723 objects degraded (12.172%) 2024-04-02T05:17:09.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:09 smithi086 ceph-mon[88839]: from='osd.2 [v2:172.21.15.86:6818/3113900521,v1:172.21.15.86:6819/3113900521]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-02T05:17:09.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:09 smithi086 ceph-mon[88839]: failed to encode map e130 with expected crc 2024-04-02T05:17:09.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:09 smithi086 ceph-mon[88839]: osdmap e130: 8 total, 7 up, 8 in 2024-04-02T05:17:09.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:09 smithi086 ceph-mon[88839]: from='osd.2 [v2:172.21.15.86:6818/3113900521,v1:172.21.15.86:6819/3113900521]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T05:17:09.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:09 smithi086 ceph-mon[88839]: failed to encode map e130 with expected crc 2024-04-02T05:17:09.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:09 smithi086 ceph-mon[88839]: failed to encode map e130 with expected crc 2024-04-02T05:17:09.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:09 smithi086 ceph-mon[88839]: from='osd.2 [v2:172.21.15.86:6818/3113900521,v1:172.21.15.86:6819/3113900521]' entity='osd.2' 2024-04-02T05:17:09.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:09.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:09.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:09 smithi086 ceph-mon[88839]: failed to encode map e130 with expected crc 2024-04-02T05:17:09.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:09 smithi086 ceph-mon[86443]: pgmap v110: 161 pgs: 18 active+undersized+degraded, 32 active+undersized, 111 active+clean; 585 KiB data, 142 MiB used, 715 GiB / 715 GiB avail; 88/723 objects degraded (12.172%) 2024-04-02T05:17:09.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:09 smithi086 ceph-mon[86443]: from='osd.2 [v2:172.21.15.86:6818/3113900521,v1:172.21.15.86:6819/3113900521]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-02T05:17:09.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:09 smithi086 ceph-mon[86443]: failed to encode map e130 with expected crc 2024-04-02T05:17:09.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:09 smithi086 ceph-mon[86443]: osdmap e130: 8 total, 7 up, 8 in 2024-04-02T05:17:09.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:09 smithi086 ceph-mon[86443]: from='osd.2 [v2:172.21.15.86:6818/3113900521,v1:172.21.15.86:6819/3113900521]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T05:17:09.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:09 smithi086 ceph-mon[86443]: failed to encode map e130 with expected crc 2024-04-02T05:17:09.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:09 smithi086 ceph-mon[86443]: failed to encode map e130 with expected crc 2024-04-02T05:17:09.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:09 smithi086 ceph-mon[86443]: from='osd.2 [v2:172.21.15.86:6818/3113900521,v1:172.21.15.86:6819/3113900521]' entity='osd.2' 2024-04-02T05:17:09.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:09.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:09.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:09 smithi086 ceph-mon[86443]: failed to encode map e130 with expected crc 2024-04-02T05:17:10.613 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:10 smithi086 ceph-mon[88839]: failed to encode map e130 with expected crc 2024-04-02T05:17:10.613 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:10 smithi086 ceph-mon[88839]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-02T05:17:10.613 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:10 smithi086 ceph-mon[88839]: osd.2 [v2:172.21.15.86:6818/3113900521,v1:172.21.15.86:6819/3113900521] boot 2024-04-02T05:17:10.613 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:10 smithi086 ceph-mon[88839]: osdmap e131: 8 total, 8 up, 8 in 2024-04-02T05:17:10.613 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:10 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T05:17:10.613 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:10 smithi086 ceph-mon[88839]: failed to encode map e131 with expected crc 2024-04-02T05:17:10.613 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:10 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:10.613 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:10 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:10.613 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:10 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:10.614 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:10 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:17:10.614 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:10 smithi086 ceph-mon[86443]: failed to encode map e130 with expected crc 2024-04-02T05:17:10.614 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:10 smithi086 ceph-mon[86443]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-02T05:17:10.614 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:10 smithi086 ceph-mon[86443]: osd.2 [v2:172.21.15.86:6818/3113900521,v1:172.21.15.86:6819/3113900521] boot 2024-04-02T05:17:10.614 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:10 smithi086 ceph-mon[86443]: osdmap e131: 8 total, 8 up, 8 in 2024-04-02T05:17:10.614 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:10 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T05:17:10.614 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:10 smithi086 ceph-mon[86443]: failed to encode map e131 with expected crc 2024-04-02T05:17:10.614 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:10 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:10.614 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:10 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:10.614 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:10 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:10.614 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:10 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:17:10.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:10 smithi192 ceph-mon[68000]: failed to encode map e130 with expected crc 2024-04-02T05:17:10.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:10 smithi192 ceph-mon[68000]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-02T05:17:10.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:10 smithi192 ceph-mon[68000]: osd.2 [v2:172.21.15.86:6818/3113900521,v1:172.21.15.86:6819/3113900521] boot 2024-04-02T05:17:10.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:10 smithi192 ceph-mon[68000]: osdmap e131: 8 total, 8 up, 8 in 2024-04-02T05:17:10.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:10 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T05:17:10.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:10 smithi192 ceph-mon[68000]: failed to encode map e131 with expected crc 2024-04-02T05:17:10.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:10 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:10.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:10 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:10.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:10 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:10.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:10 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:17:11.616 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:11 smithi086 ceph-mon[86443]: failed to encode map e131 with expected crc 2024-04-02T05:17:11.616 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:11 smithi086 ceph-mon[86443]: failed to encode map e131 with expected crc 2024-04-02T05:17:11.616 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:11 smithi086 ceph-mon[86443]: failed to encode map e131 with expected crc 2024-04-02T05:17:11.616 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:11 smithi086 ceph-mon[86443]: failed to encode map e131 with expected crc 2024-04-02T05:17:11.616 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:11 smithi086 ceph-mon[86443]: failed to encode map e131 with expected crc 2024-04-02T05:17:11.616 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:11 smithi086 ceph-mon[86443]: pgmap v113: 161 pgs: 18 active+undersized+degraded, 32 active+undersized, 111 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 88/723 objects degraded (12.172%) 2024-04-02T05:17:11.616 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:11 smithi086 ceph-mon[86443]: osdmap e132: 8 total, 8 up, 8 in 2024-04-02T05:17:11.616 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:11 smithi086 ceph-mon[86443]: failed to encode map e132 with expected crc 2024-04-02T05:17:11.616 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:11 smithi086 ceph-mon[86443]: failed to encode map e132 with expected crc 2024-04-02T05:17:11.617 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:11 smithi086 ceph-mon[86443]: failed to encode map e132 with expected crc 2024-04-02T05:17:11.617 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:11 smithi086 ceph-mon[86443]: failed to encode map e132 with expected crc 2024-04-02T05:17:11.617 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:11 smithi086 ceph-mon[86443]: failed to encode map e132 with expected crc 2024-04-02T05:17:11.617 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:11 smithi086 ceph-mon[88839]: failed to encode map e131 with expected crc 2024-04-02T05:17:11.617 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:11 smithi086 ceph-mon[88839]: failed to encode map e131 with expected crc 2024-04-02T05:17:11.617 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:11 smithi086 ceph-mon[88839]: failed to encode map e131 with expected crc 2024-04-02T05:17:11.617 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:11 smithi086 ceph-mon[88839]: failed to encode map e131 with expected crc 2024-04-02T05:17:11.617 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:11 smithi086 ceph-mon[88839]: failed to encode map e131 with expected crc 2024-04-02T05:17:11.617 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:11 smithi086 ceph-mon[88839]: pgmap v113: 161 pgs: 18 active+undersized+degraded, 32 active+undersized, 111 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 88/723 objects degraded (12.172%) 2024-04-02T05:17:11.617 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:11 smithi086 ceph-mon[88839]: osdmap e132: 8 total, 8 up, 8 in 2024-04-02T05:17:11.617 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:11 smithi086 ceph-mon[88839]: failed to encode map e132 with expected crc 2024-04-02T05:17:11.617 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:11 smithi086 ceph-mon[88839]: failed to encode map e132 with expected crc 2024-04-02T05:17:11.618 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:11 smithi086 ceph-mon[88839]: failed to encode map e132 with expected crc 2024-04-02T05:17:11.618 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:11 smithi086 ceph-mon[88839]: failed to encode map e132 with expected crc 2024-04-02T05:17:11.618 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:11 smithi086 ceph-mon[88839]: failed to encode map e132 with expected crc 2024-04-02T05:17:11.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:11 smithi192 ceph-mon[68000]: failed to encode map e131 with expected crc 2024-04-02T05:17:11.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:11 smithi192 ceph-mon[68000]: failed to encode map e131 with expected crc 2024-04-02T05:17:11.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:11 smithi192 ceph-mon[68000]: failed to encode map e131 with expected crc 2024-04-02T05:17:11.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:11 smithi192 ceph-mon[68000]: failed to encode map e131 with expected crc 2024-04-02T05:17:11.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:11 smithi192 ceph-mon[68000]: failed to encode map e131 with expected crc 2024-04-02T05:17:11.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:11 smithi192 ceph-mon[68000]: pgmap v113: 161 pgs: 18 active+undersized+degraded, 32 active+undersized, 111 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 88/723 objects degraded (12.172%) 2024-04-02T05:17:11.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:11 smithi192 ceph-mon[68000]: osdmap e132: 8 total, 8 up, 8 in 2024-04-02T05:17:11.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:11 smithi192 ceph-mon[68000]: failed to encode map e132 with expected crc 2024-04-02T05:17:11.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:11 smithi192 ceph-mon[68000]: failed to encode map e132 with expected crc 2024-04-02T05:17:11.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:11 smithi192 ceph-mon[68000]: failed to encode map e132 with expected crc 2024-04-02T05:17:11.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:11 smithi192 ceph-mon[68000]: failed to encode map e132 with expected crc 2024-04-02T05:17:11.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:11 smithi192 ceph-mon[68000]: failed to encode map e132 with expected crc 2024-04-02T05:17:12.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:12 smithi086 ceph-mon[88839]: failed to encode map e132 with expected crc 2024-04-02T05:17:12.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:12 smithi086 ceph-mon[88839]: failed to encode map e132 with expected crc 2024-04-02T05:17:12.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:12 smithi086 ceph-mon[88839]: failed to encode map e132 with expected crc 2024-04-02T05:17:12.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:12 smithi086 ceph-mon[88839]: failed to encode map e132 with expected crc 2024-04-02T05:17:12.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:12 smithi086 ceph-mon[88839]: failed to encode map e132 with expected crc 2024-04-02T05:17:12.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:12 smithi086 ceph-mon[88839]: failed to encode map e132 with expected crc 2024-04-02T05:17:12.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:12 smithi086 ceph-mon[88839]: failed to encode map e132 with expected crc 2024-04-02T05:17:12.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:12 smithi086 ceph-mon[88839]: failed to encode map e132 with expected crc 2024-04-02T05:17:12.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:12 smithi086 ceph-mon[88839]: failed to encode map e132 with expected crc 2024-04-02T05:17:12.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:12 smithi086 ceph-mon[88839]: Health check update: Degraded data redundancy: 88/723 objects degraded (12.172%), 18 pgs degraded (PG_DEGRADED) 2024-04-02T05:17:12.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:12 smithi086 ceph-mon[86443]: failed to encode map e132 with expected crc 2024-04-02T05:17:12.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:12 smithi086 ceph-mon[86443]: failed to encode map e132 with expected crc 2024-04-02T05:17:12.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:12 smithi086 ceph-mon[86443]: failed to encode map e132 with expected crc 2024-04-02T05:17:12.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:12 smithi086 ceph-mon[86443]: failed to encode map e132 with expected crc 2024-04-02T05:17:12.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:12 smithi086 ceph-mon[86443]: failed to encode map e132 with expected crc 2024-04-02T05:17:12.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:12 smithi086 ceph-mon[86443]: failed to encode map e132 with expected crc 2024-04-02T05:17:12.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:12 smithi086 ceph-mon[86443]: failed to encode map e132 with expected crc 2024-04-02T05:17:12.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:12 smithi086 ceph-mon[86443]: failed to encode map e132 with expected crc 2024-04-02T05:17:12.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:12 smithi086 ceph-mon[86443]: failed to encode map e132 with expected crc 2024-04-02T05:17:12.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:12 smithi086 ceph-mon[86443]: Health check update: Degraded data redundancy: 88/723 objects degraded (12.172%), 18 pgs degraded (PG_DEGRADED) 2024-04-02T05:17:12.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:12 smithi192 ceph-mon[68000]: failed to encode map e132 with expected crc 2024-04-02T05:17:12.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:12 smithi192 ceph-mon[68000]: failed to encode map e132 with expected crc 2024-04-02T05:17:12.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:12 smithi192 ceph-mon[68000]: failed to encode map e132 with expected crc 2024-04-02T05:17:12.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:12 smithi192 ceph-mon[68000]: failed to encode map e132 with expected crc 2024-04-02T05:17:12.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:12 smithi192 ceph-mon[68000]: failed to encode map e132 with expected crc 2024-04-02T05:17:12.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:12 smithi192 ceph-mon[68000]: failed to encode map e132 with expected crc 2024-04-02T05:17:12.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:12 smithi192 ceph-mon[68000]: failed to encode map e132 with expected crc 2024-04-02T05:17:12.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:12 smithi192 ceph-mon[68000]: failed to encode map e132 with expected crc 2024-04-02T05:17:12.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:12 smithi192 ceph-mon[68000]: failed to encode map e132 with expected crc 2024-04-02T05:17:12.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:12 smithi192 ceph-mon[68000]: Health check update: Degraded data redundancy: 88/723 objects degraded (12.172%), 18 pgs degraded (PG_DEGRADED) 2024-04-02T05:17:13.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:13 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:17:13.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:13 smithi192 ceph-mon[68000]: pgmap v115: 161 pgs: 6 active+undersized+degraded, 14 active+undersized, 141 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 37/723 objects degraded (5.118%) 2024-04-02T05:17:13.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:17:13 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:17:13] "GET /metrics HTTP/1.1" 200 38038 "" "Prometheus/2.43.0" 2024-04-02T05:17:13.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:13 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:17:13.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:13 smithi086 ceph-mon[88839]: pgmap v115: 161 pgs: 6 active+undersized+degraded, 14 active+undersized, 141 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 37/723 objects degraded (5.118%) 2024-04-02T05:17:13.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:13 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:17:13.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:13 smithi086 ceph-mon[86443]: pgmap v115: 161 pgs: 6 active+undersized+degraded, 14 active+undersized, 141 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 37/723 objects degraded (5.118%) 2024-04-02T05:17:14.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:14.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:14.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:17:14.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:17:14.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:14.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:17:14.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:14.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:14.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all nvmeof 2024-04-02T05:17:14.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:14.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:14.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:14.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:14.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:14.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:14.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:14.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: Upgrade: Finalizing container_image settings 2024-04-02T05:17:14.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:17:14.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:17:14.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-02T05:17:14.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-02T05:17:14.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:17:14.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:17:14.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-02T05:17:14.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:17:14.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:17:14.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:14.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:14.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:17:14.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:17:14.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:14.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:17:14.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:14.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:14.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all nvmeof 2024-04-02T05:17:14.836 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:14.837 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:14.837 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:14.837 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:14.837 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:14.837 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:14.837 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:14.837 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: Upgrade: Finalizing container_image settings 2024-04-02T05:17:14.837 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:17:14.837 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:17:14.837 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-02T05:17:14.837 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.837 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.837 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-02T05:17:14.837 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:17:14.838 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:17:14.838 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-02T05:17:14.838 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:17:14.838 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:17:14.838 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:17:14.838 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:17:14.838 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:17:14.838 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:17:14.839 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:17:14.839 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:17:14.839 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.839 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.839 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:17:14.839 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:17:14.839 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:17:14.839 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:17:14.839 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:17:14.839 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:17:14.839 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:17:14.839 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:17:14.839 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-02T05:17:14.840 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.840 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.840 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.840 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.840 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.840 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.840 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.840 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.840 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.840 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.840 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.841 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.841 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: Upgrade: Complete! 2024-04-02T05:17:14.841 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:17:14.841 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:17:14.841 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-02T05:17:14.841 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:17:14.841 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:17:14.841 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:14.841 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:17:14.841 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:17:14.841 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:14.841 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[86443]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:17:14.842 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:17:14.842 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:17:14.842 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:17:14.842 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:17:14.842 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:17:14.842 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:17:14.842 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.842 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.842 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:17:14.843 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:17:14.843 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:17:14.843 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:17:14.843 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:17:14.843 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:17:14.843 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:17:14.843 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:17:14.843 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-02T05:17:14.843 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.843 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.843 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.843 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.843 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.843 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.844 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.844 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.844 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.844 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.844 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.844 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.844 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: Upgrade: Complete! 2024-04-02T05:17:14.844 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:17:14.844 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:17:14.844 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-02T05:17:14.844 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:17:14.844 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:17:14.844 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:14.844 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:17:14.844 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:17:14.845 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:14.845 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:14 smithi086 ceph-mon[88839]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:17:14.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:14.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:14.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:17:14.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:17:14.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:14.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:17:14.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:14.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:14.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all nvmeof 2024-04-02T05:17:14.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:14.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:14.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:14.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:14.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:14.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:14.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:14.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: Upgrade: Finalizing container_image settings 2024-04-02T05:17:14.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:17:14.930 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:17:14.930 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-02T05:17:14.930 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.930 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.930 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-02T05:17:14.930 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:17:14.930 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:17:14.930 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-02T05:17:14.930 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:17:14.931 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:17:14.931 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:17:14.931 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:17:14.931 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:17:14.931 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:17:14.931 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:17:14.932 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:17:14.932 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.932 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.932 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:17:14.932 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:17:14.932 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:17:14.932 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:17:14.932 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:17:14.933 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:17:14.933 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:17:14.933 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:17:14.933 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-02T05:17:14.933 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.933 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.933 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.933 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.933 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.933 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.933 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.934 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.934 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.934 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.934 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.934 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:17:14.934 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: Upgrade: Complete! 2024-04-02T05:17:14.934 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:17:14.934 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:17:14.934 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-02T05:17:14.934 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:17:14.934 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:17:14.934 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:14.934 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:17:14.935 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:17:14.935 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:14.935 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:14 smithi192 ceph-mon[68000]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:17:15.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:15 smithi086 ceph-mon[86443]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 37/723 objects degraded (5.118%), 6 pgs degraded) 2024-04-02T05:17:15.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:15 smithi086 ceph-mon[86443]: Cluster is now healthy 2024-04-02T05:17:15.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:15 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:15.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:15 smithi086 ceph-mon[88839]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 37/723 objects degraded (5.118%), 6 pgs degraded) 2024-04-02T05:17:15.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:15 smithi086 ceph-mon[88839]: Cluster is now healthy 2024-04-02T05:17:15.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:15 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:15.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:15 smithi192 ceph-mon[68000]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 37/723 objects degraded (5.118%), 6 pgs degraded) 2024-04-02T05:17:15.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:15 smithi192 ceph-mon[68000]: Cluster is now healthy 2024-04-02T05:17:15.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:15 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:16.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:16 smithi086 ceph-mon[86443]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-02T05:17:16.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:16 smithi086 ceph-mon[88839]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-02T05:17:16.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:16 smithi192 ceph-mon[68000]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-02T05:17:19.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:19 smithi192 ceph-mon[68000]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:17:19.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:19 smithi086 ceph-mon[86443]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:17:19.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:19 smithi086 ceph-mon[88839]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:17:21.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:21 smithi192 ceph-mon[68000]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:17:21.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:21 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:21.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:21 smithi086 ceph-mon[88839]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:17:21.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:21 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:21.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:21 smithi086 ceph-mon[86443]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:17:21.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:21 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:23.636 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:23 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:17:23.636 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:23 smithi086 ceph-mon[86443]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-02T05:17:23.636 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:17:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:17:23] "GET /metrics HTTP/1.1" 200 38038 "" "Prometheus/2.43.0" 2024-04-02T05:17:23.636 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:23 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:17:23.636 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:23 smithi086 ceph-mon[88839]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-02T05:17:23.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:23 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:17:23.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:23 smithi192 ceph-mon[68000]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-02T05:17:25.297 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch ps' 2024-04-02T05:17:25.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:25 smithi086 ceph-mon[86443]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:17:25.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:25 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:25.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:25 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:17:25.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:25 smithi086 ceph-mon[88839]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:17:25.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:25 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:25.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:25 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:17:25.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:25 smithi192 ceph-mon[68000]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:17:25.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:25 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:25.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:25 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:17:26.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:26 smithi086 ceph-mon[86443]: from='client.64306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:17:26.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:26 smithi086 ceph-mon[88839]: from='client.64306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:17:26.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:26 smithi192 ceph-mon[68000]: from='client.64306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:17:27.414 INFO:teuthology.orchestra.run.smithi086.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T05:17:27.414 INFO:teuthology.orchestra.run.smithi086.stdout:alertmanager.a smithi086 *:9093,9094 running (5m) 18s ago 27m 26.3M - 0.25.0 c8568f914cd2 660ea62bb6b9 2024-04-02T05:17:27.414 INFO:teuthology.orchestra.run.smithi086.stdout:grafana.a smithi192 *:3000 running (5m) 3m ago 27m 79.2M - 9.4.12 2bacad6d85d8 259af9e63aa5 2024-04-02T05:17:27.414 INFO:teuthology.orchestra.run.smithi086.stdout:iscsi.foo.smithi086.rsbqwx smithi086 running (4m) 18s ago 26m 59.9M - 3.5 e1d6a67b021e fe728eb940f7 2024-04-02T05:17:27.414 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.x smithi192 *:8443,9283,8765 running (6m) 3m ago 31m 449M - 19.0.0-1979-g6cec793d 78f830190839 dd1e6d0f0a95 2024-04-02T05:17:27.414 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.y smithi086 *:8443,9283,8765 running (16m) 18s ago 33m 505M - 19.0.0-1979-g6cec793d 78f830190839 3dca54a018ab 2024-04-02T05:17:27.414 INFO:teuthology.orchestra.run.smithi086.stdout:mon.a smithi086 running (3m) 18s ago 33m 50.0M 2048M 19.0.0-1979-g6cec793d 78f830190839 8e478dd4da2d 2024-04-02T05:17:27.415 INFO:teuthology.orchestra.run.smithi086.stdout:mon.b smithi192 running (4m) 3m ago 32m 36.2M 2048M 19.0.0-1979-g6cec793d 78f830190839 4125e997188f 2024-04-02T05:17:27.415 INFO:teuthology.orchestra.run.smithi086.stdout:mon.c smithi086 running (3m) 18s ago 32m 40.5M 2048M 19.0.0-1979-g6cec793d 78f830190839 8180999b3142 2024-04-02T05:17:27.415 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.a smithi086 *:9100 running (6m) 18s ago 28m 15.9M - 1.5.0 0da6a335fe13 05a844569d7b 2024-04-02T05:17:27.415 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.b smithi192 *:9100 running (6m) 3m ago 28m 17.4M - 1.5.0 0da6a335fe13 1a7d6788d58b 2024-04-02T05:17:27.415 INFO:teuthology.orchestra.run.smithi086.stdout:osd.0 smithi086 running (109s) 18s ago 31m 44.3M 2992M 19.0.0-1979-g6cec793d 78f830190839 e521a0ef7daf 2024-04-02T05:17:27.415 INFO:teuthology.orchestra.run.smithi086.stdout:osd.1 smithi086 running (74s) 18s ago 31m 51.5M 2992M 19.0.0-1979-g6cec793d 78f830190839 76fe72844771 2024-04-02T05:17:27.415 INFO:teuthology.orchestra.run.smithi086.stdout:osd.2 smithi086 running (23s) 18s ago 30m 12.2M 2992M 19.0.0-1979-g6cec793d 78f830190839 28594f089a8f 2024-04-02T05:17:27.415 INFO:teuthology.orchestra.run.smithi086.stdout:osd.3 smithi086 running (30m) 18s ago 30m 57.1M 2992M 17.2.0 e1d6a67b021e 327538dac44e 2024-04-02T05:17:27.415 INFO:teuthology.orchestra.run.smithi086.stdout:osd.4 smithi192 running (30m) 3m ago 30m 56.1M 3248M 17.2.0 e1d6a67b021e 6535d7919a2d 2024-04-02T05:17:27.415 INFO:teuthology.orchestra.run.smithi086.stdout:osd.5 smithi192 running (29m) 3m ago 29m 54.9M 3248M 17.2.0 e1d6a67b021e 7259249605f2 2024-04-02T05:17:27.415 INFO:teuthology.orchestra.run.smithi086.stdout:osd.6 smithi192 running (29m) 3m ago 29m 52.9M 3248M 17.2.0 e1d6a67b021e bb7d171b02a1 2024-04-02T05:17:27.415 INFO:teuthology.orchestra.run.smithi086.stdout:osd.7 smithi192 running (28m) 3m ago 28m 54.9M 3248M 17.2.0 e1d6a67b021e 95f5a0353999 2024-04-02T05:17:27.415 INFO:teuthology.orchestra.run.smithi086.stdout:prometheus.a smithi192 *:9095 running (5m) 3m ago 28m 65.1M - 2.43.0 a07b618ecd1d 8378fcdb60e3 2024-04-02T05:17:27.415 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi086.bxzxlp smithi086 *:8000 running (26m) 18s ago 26m 79.8M - 17.2.0 e1d6a67b021e 6725785fbc53 2024-04-02T05:17:27.416 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi192.pmlbot smithi192 *:8000 running (26m) 3m ago 26m 77.9M - 17.2.0 e1d6a67b021e 5ae3638f8b1c 2024-04-02T05:17:27.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:27 smithi192 ceph-mon[68000]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:17:27.711 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:27 smithi086 ceph-mon[86443]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:17:27.712 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:27 smithi086 ceph-mon[88839]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:17:28.119 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 2'"'"'' 2024-04-02T05:17:29.386 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:29 smithi086 ceph-mon[88839]: from='client.64312 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:17:29.386 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:29 smithi086 ceph-mon[88839]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:17:29.386 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:29 smithi086 ceph-mon[86443]: from='client.64312 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:17:29.386 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:29 smithi086 ceph-mon[86443]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:17:29.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:29 smithi192 ceph-mon[68000]: from='client.64312 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:17:29.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:29 smithi192 ceph-mon[68000]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:17:30.161 INFO:teuthology.orchestra.run.smithi086.stdout:true 2024-04-02T05:17:30.514 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:30 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/3826958647' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:30.514 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:30 smithi086 ceph-mon[88839]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:17:30.515 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:30 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/3826958647' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:30.515 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:30 smithi086 ceph-mon[86443]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:17:30.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:30 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/3826958647' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:30.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:30 smithi192 ceph-mon[68000]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:17:30.792 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 8'"'"'' 2024-04-02T05:17:33.656 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:17:33 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:17:33] "GET /metrics HTTP/1.1" 200 38185 "" "Prometheus/2.43.0" 2024-04-02T05:17:33.657 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:33 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:17:33.657 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:33 smithi086 ceph-mon[88839]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:17:33.657 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:33 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:17:33.657 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:33 smithi086 ceph-mon[86443]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:17:33.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:33 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:17:33.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:33 smithi192 ceph-mon[68000]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:17:34.317 INFO:teuthology.orchestra.run.smithi086.stdout:true 2024-04-02T05:17:34.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:34 smithi086 ceph-mon[88839]: from='client.64324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:17:34.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:34 smithi086 ceph-mon[86443]: from='client.64324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:17:34.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:34 smithi192 ceph-mon[68000]: from='client.64324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:17:35.188 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch upgrade status' 2024-04-02T05:17:35.495 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:35 smithi086 ceph-mon[86443]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:17:35.495 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:35 smithi086 ceph-mon[88839]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:17:35.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:35 smithi192 ceph-mon[68000]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:17:37.213 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:17:37.214 INFO:teuthology.orchestra.run.smithi086.stdout: "target_image": null, 2024-04-02T05:17:37.214 INFO:teuthology.orchestra.run.smithi086.stdout: "in_progress": false, 2024-04-02T05:17:37.214 INFO:teuthology.orchestra.run.smithi086.stdout: "which": "", 2024-04-02T05:17:37.214 INFO:teuthology.orchestra.run.smithi086.stdout: "services_complete": [], 2024-04-02T05:17:37.214 INFO:teuthology.orchestra.run.smithi086.stdout: "progress": null, 2024-04-02T05:17:37.214 INFO:teuthology.orchestra.run.smithi086.stdout: "message": "", 2024-04-02T05:17:37.214 INFO:teuthology.orchestra.run.smithi086.stdout: "is_paused": false 2024-04-02T05:17:37.214 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:17:37.527 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:37 smithi086 ceph-mon[88839]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:17:37.527 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:37 smithi086 ceph-mon[86443]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:17:37.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:37 smithi192 ceph-mon[68000]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:17:37.821 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph health detail' 2024-04-02T05:17:38.447 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:38 smithi086 ceph-mon[86443]: from='client.54387 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:17:38.448 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:38 smithi086 ceph-mon[88839]: from='client.54387 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:17:38.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:38 smithi192 ceph-mon[68000]: from='client.54387 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:17:39.462 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:39 smithi086 ceph-mon[88839]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:17:39.462 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:39 smithi086 ceph-mon[86443]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:17:39.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:39 smithi192 ceph-mon[68000]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:17:39.981 INFO:teuthology.orchestra.run.smithi086.stdout:HEALTH_OK 2024-04-02T05:17:40.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:40 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/3757983143' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T05:17:40.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:40 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:17:40.742 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:40 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/3757983143' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T05:17:40.742 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:40 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:17:40.742 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:40 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/3757983143' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T05:17:40.742 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:40 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:17:40.767 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd' 2024-04-02T05:17:41.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:41 smithi192 ceph-mon[68000]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:17:41.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:41 smithi086 ceph-mon[86443]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:17:41.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:41 smithi086 ceph-mon[88839]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:17:42.536 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:42 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:17:42.536 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:42 smithi086 ceph-mon[86443]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:17:42.536 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:42 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:17:42.536 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:42 smithi086 ceph-mon[88839]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:17:42.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:42 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:17:42.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:42 smithi192 ceph-mon[68000]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:17:43.572 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:43 smithi086 ceph-mon[88839]: from='client.64342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:17:43.572 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:17:43 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:17:43] "GET /metrics HTTP/1.1" 200 38179 "" "Prometheus/2.43.0" 2024-04-02T05:17:43.572 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:43 smithi086 ceph-mon[86443]: from='client.64342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:17:43.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:43 smithi192 ceph-mon[68000]: from='client.64342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:17:44.334 INFO:teuthology.orchestra.run.smithi086.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:17:45.170 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-02T05:17:45.398 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:45 smithi086 ceph-mon[88839]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:17:45.399 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:45 smithi086 ceph-mon[88839]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:17:45.399 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:45 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:45.399 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:45 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:17:45.399 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:45 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:17:45.399 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:45 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:45.399 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:45 smithi086 ceph-mon[86443]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:17:45.399 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:45 smithi086 ceph-mon[86443]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:17:45.399 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:45 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:45.399 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:45 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:17:45.399 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:45 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:17:45.400 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:45 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:45.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:45 smithi192 ceph-mon[68000]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:17:45.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:45 smithi192 ceph-mon[68000]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:17:45.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:45 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:45.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:45 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:17:45.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:45 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:17:45.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:45 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:46.619 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:46 smithi086 ceph-mon[88839]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:17:46.619 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:46 smithi086 ceph-mon[86443]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:17:46.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:46 smithi192 ceph-mon[68000]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:17:47.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:47 smithi192 ceph-mon[68000]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:17:47.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:47 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:47.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:47 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:17:47.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:47 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:47.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:47 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:47.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:47 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:47.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:47 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:47.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:47 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:47.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:47 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:47.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:47 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-02T05:17:47.712 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:47 smithi086 ceph-mon[88839]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:17:47.712 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:47 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:47.712 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:47 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:17:47.712 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:47 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:47.712 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:47 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:47.713 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:47 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:47.713 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:47 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:47.713 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:47 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:47.713 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:47 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:47.713 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:47 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-02T05:17:47.713 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:47 smithi086 ceph-mon[86443]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:17:47.713 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:47 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:47.713 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:47 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:17:47.713 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:47 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:47.713 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:47 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:47.713 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:47 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:47.713 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:47 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:47.713 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:47 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:47.713 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:47 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:47.714 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:47 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-02T05:17:48.449 INFO:teuthology.orchestra.run.smithi086.stdout:true 2024-04-02T05:17:48.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:48 smithi086 ceph-mon[86443]: Upgrade: Target is version 19.0.0-1979-g6cec793d (squid) 2024-04-02T05:17:48.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:48 smithi086 ceph-mon[86443]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7bd5b84926d4a0d6c51e5562e965bff211cec2740c9f4dec9b5e00a2bbb75eae'] 2024-04-02T05:17:48.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:48 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all mgr 2024-04-02T05:17:48.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:48 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all mon 2024-04-02T05:17:48.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:48 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all crash 2024-04-02T05:17:48.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:48 smithi086 ceph-mon[86443]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-02T05:17:48.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:48 smithi086 ceph-mon[86443]: Upgrade: osd.3 is safe to restart 2024-04-02T05:17:48.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:48 smithi086 ceph-mon[88839]: Upgrade: Target is version 19.0.0-1979-g6cec793d (squid) 2024-04-02T05:17:48.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:48 smithi086 ceph-mon[88839]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7bd5b84926d4a0d6c51e5562e965bff211cec2740c9f4dec9b5e00a2bbb75eae'] 2024-04-02T05:17:48.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:48 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all mgr 2024-04-02T05:17:48.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:48 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all mon 2024-04-02T05:17:48.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:48 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all crash 2024-04-02T05:17:48.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:48 smithi086 ceph-mon[88839]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-02T05:17:48.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:48 smithi086 ceph-mon[88839]: Upgrade: osd.3 is safe to restart 2024-04-02T05:17:48.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:48 smithi192 ceph-mon[68000]: Upgrade: Target is version 19.0.0-1979-g6cec793d (squid) 2024-04-02T05:17:48.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:48 smithi192 ceph-mon[68000]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7bd5b84926d4a0d6c51e5562e965bff211cec2740c9f4dec9b5e00a2bbb75eae'] 2024-04-02T05:17:48.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:48 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all mgr 2024-04-02T05:17:48.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:48 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all mon 2024-04-02T05:17:48.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:48 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all crash 2024-04-02T05:17:48.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:48 smithi192 ceph-mon[68000]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-02T05:17:48.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:48 smithi192 ceph-mon[68000]: Upgrade: osd.3 is safe to restart 2024-04-02T05:17:49.109 INFO:teuthology.orchestra.run.smithi086.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T05:17:49.109 INFO:teuthology.orchestra.run.smithi086.stdout:alertmanager.a smithi086 *:9093,9094 running (5m) 40s ago 27m 26.3M - 0.25.0 c8568f914cd2 660ea62bb6b9 2024-04-02T05:17:49.109 INFO:teuthology.orchestra.run.smithi086.stdout:grafana.a smithi192 *:3000 running (5m) 3m ago 27m 79.2M - 9.4.12 2bacad6d85d8 259af9e63aa5 2024-04-02T05:17:49.109 INFO:teuthology.orchestra.run.smithi086.stdout:iscsi.foo.smithi086.rsbqwx smithi086 running (5m) 40s ago 26m 59.9M - 3.5 e1d6a67b021e fe728eb940f7 2024-04-02T05:17:49.109 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.x smithi192 *:8443,9283,8765 running (6m) 3m ago 32m 449M - 19.0.0-1979-g6cec793d 78f830190839 dd1e6d0f0a95 2024-04-02T05:17:49.109 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.y smithi086 *:8443,9283,8765 running (16m) 40s ago 33m 505M - 19.0.0-1979-g6cec793d 78f830190839 3dca54a018ab 2024-04-02T05:17:49.110 INFO:teuthology.orchestra.run.smithi086.stdout:mon.a smithi086 running (3m) 40s ago 33m 50.0M 2048M 19.0.0-1979-g6cec793d 78f830190839 8e478dd4da2d 2024-04-02T05:17:49.110 INFO:teuthology.orchestra.run.smithi086.stdout:mon.b smithi192 running (4m) 3m ago 32m 36.2M 2048M 19.0.0-1979-g6cec793d 78f830190839 4125e997188f 2024-04-02T05:17:49.110 INFO:teuthology.orchestra.run.smithi086.stdout:mon.c smithi086 running (3m) 40s ago 32m 40.5M 2048M 19.0.0-1979-g6cec793d 78f830190839 8180999b3142 2024-04-02T05:17:49.110 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.a smithi086 *:9100 running (6m) 40s ago 28m 15.9M - 1.5.0 0da6a335fe13 05a844569d7b 2024-04-02T05:17:49.110 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.b smithi192 *:9100 running (6m) 3m ago 28m 17.4M - 1.5.0 0da6a335fe13 1a7d6788d58b 2024-04-02T05:17:49.110 INFO:teuthology.orchestra.run.smithi086.stdout:osd.0 smithi086 running (2m) 40s ago 31m 44.3M 2992M 19.0.0-1979-g6cec793d 78f830190839 e521a0ef7daf 2024-04-02T05:17:49.110 INFO:teuthology.orchestra.run.smithi086.stdout:osd.1 smithi086 running (96s) 40s ago 31m 51.5M 2992M 19.0.0-1979-g6cec793d 78f830190839 76fe72844771 2024-04-02T05:17:49.110 INFO:teuthology.orchestra.run.smithi086.stdout:osd.2 smithi086 running (45s) 40s ago 31m 12.2M 2992M 19.0.0-1979-g6cec793d 78f830190839 28594f089a8f 2024-04-02T05:17:49.110 INFO:teuthology.orchestra.run.smithi086.stdout:osd.3 smithi086 running (30m) 40s ago 30m 57.1M 2992M 17.2.0 e1d6a67b021e 327538dac44e 2024-04-02T05:17:49.110 INFO:teuthology.orchestra.run.smithi086.stdout:osd.4 smithi192 running (30m) 3m ago 30m 56.1M 3248M 17.2.0 e1d6a67b021e 6535d7919a2d 2024-04-02T05:17:49.110 INFO:teuthology.orchestra.run.smithi086.stdout:osd.5 smithi192 running (30m) 3m ago 30m 54.9M 3248M 17.2.0 e1d6a67b021e 7259249605f2 2024-04-02T05:17:49.110 INFO:teuthology.orchestra.run.smithi086.stdout:osd.6 smithi192 running (29m) 3m ago 29m 52.9M 3248M 17.2.0 e1d6a67b021e bb7d171b02a1 2024-04-02T05:17:49.110 INFO:teuthology.orchestra.run.smithi086.stdout:osd.7 smithi192 running (29m) 3m ago 29m 54.9M 3248M 17.2.0 e1d6a67b021e 95f5a0353999 2024-04-02T05:17:49.110 INFO:teuthology.orchestra.run.smithi086.stdout:prometheus.a smithi192 *:9095 running (6m) 3m ago 28m 65.1M - 2.43.0 a07b618ecd1d 8378fcdb60e3 2024-04-02T05:17:49.110 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi086.bxzxlp smithi086 *:8000 running (27m) 40s ago 27m 79.8M - 17.2.0 e1d6a67b021e 6725785fbc53 2024-04-02T05:17:49.111 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi192.pmlbot smithi192 *:8000 running (27m) 3m ago 27m 77.9M - 17.2.0 e1d6a67b021e 5ae3638f8b1c 2024-04-02T05:17:49.407 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:49 smithi086 ceph-mon[88839]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:17:49.408 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:49 smithi086 ceph-mon[86443]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:17:49.527 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:17:49.527 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": { 2024-04-02T05:17:49.527 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 3 2024-04-02T05:17:49.527 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:17:49.527 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": { 2024-04-02T05:17:49.527 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 2 2024-04-02T05:17:49.527 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:17:49.527 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": { 2024-04-02T05:17:49.528 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-04-02T05:17:49.528 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 3 2024-04-02T05:17:49.528 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:17:49.528 INFO:teuthology.orchestra.run.smithi086.stdout: "rgw": { 2024-04-02T05:17:49.528 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-02T05:17:49.528 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:17:49.528 INFO:teuthology.orchestra.run.smithi086.stdout: "overall": { 2024-04-02T05:17:49.528 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-04-02T05:17:49.528 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 8 2024-04-02T05:17:49.528 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-04-02T05:17:49.528 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:17:49.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:49 smithi192 ceph-mon[68000]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:17:49.874 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:17:49.875 INFO:teuthology.orchestra.run.smithi086.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", 2024-04-02T05:17:49.875 INFO:teuthology.orchestra.run.smithi086.stdout: "in_progress": true, 2024-04-02T05:17:49.875 INFO:teuthology.orchestra.run.smithi086.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-04-02T05:17:49.875 INFO:teuthology.orchestra.run.smithi086.stdout: "services_complete": [], 2024-04-02T05:17:49.875 INFO:teuthology.orchestra.run.smithi086.stdout: "progress": "3/8 daemons upgraded", 2024-04-02T05:17:49.875 INFO:teuthology.orchestra.run.smithi086.stdout: "message": "Currently upgrading osd daemons", 2024-04-02T05:17:49.875 INFO:teuthology.orchestra.run.smithi086.stdout: "is_paused": false 2024-04-02T05:17:49.875 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:17:50.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:50 smithi086 ceph-mon[86443]: from='client.64348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:17:50.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:50 smithi086 ceph-mon[86443]: from='client.54411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:17:50.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:50 smithi086 ceph-mon[86443]: from='client.54417 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:17:50.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:50 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:50.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:50 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-02T05:17:50.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:50 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:17:50.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:50 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/3835264948' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:50.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:50 smithi086 ceph-mon[88839]: from='client.64348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:17:50.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:50 smithi086 ceph-mon[88839]: from='client.54411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:17:50.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:50 smithi086 ceph-mon[88839]: from='client.54417 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:17:50.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:50 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:50.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:50 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-02T05:17:50.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:50 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:17:50.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:50 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/3835264948' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:50.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:50 smithi192 ceph-mon[68000]: from='client.64348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:17:50.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:50 smithi192 ceph-mon[68000]: from='client.54411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:17:50.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:50 smithi192 ceph-mon[68000]: from='client.54417 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:17:50.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:50 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:50.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:50 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-02T05:17:50.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:50 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:17:50.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:50 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/3835264948' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:17:51.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:51 smithi086 ceph-mon[86443]: Upgrade: Updating osd.3 2024-04-02T05:17:51.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:51 smithi086 ceph-mon[86443]: Deploying daemon osd.3 on smithi086 2024-04-02T05:17:51.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:51 smithi086 ceph-mon[86443]: from='client.64372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:17:51.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:51 smithi086 ceph-mon[86443]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:17:51.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:51 smithi086 ceph-mon[88839]: Upgrade: Updating osd.3 2024-04-02T05:17:51.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:51 smithi086 ceph-mon[88839]: Deploying daemon osd.3 on smithi086 2024-04-02T05:17:51.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:51 smithi086 ceph-mon[88839]: from='client.64372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:17:51.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:51 smithi086 ceph-mon[88839]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:17:51.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:51 smithi192 ceph-mon[68000]: Upgrade: Updating osd.3 2024-04-02T05:17:51.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:51 smithi192 ceph-mon[68000]: Deploying daemon osd.3 on smithi086 2024-04-02T05:17:51.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:51 smithi192 ceph-mon[68000]: from='client.64372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:17:51.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:51 smithi192 ceph-mon[68000]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:17:53.083 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:52 smithi086 systemd[1]: Stopping Ceph osd.3 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:17:53.083 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:52 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3[41227]: 2024-04-02T05:17:52.915+0000 7fe452443700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T05:17:53.083 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:52 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3[41227]: 2024-04-02T05:17:52.915+0000 7fe452443700 -1 osd.3 132 *** Got signal Terminated *** 2024-04-02T05:17:53.083 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:52 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3[41227]: 2024-04-02T05:17:52.915+0000 7fe452443700 -1 osd.3 132 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T05:17:53.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:53 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:17:53.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:53 smithi192 ceph-mon[68000]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:17:53.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:53 smithi192 ceph-mon[68000]: osd.3 marked itself down and dead 2024-04-02T05:17:53.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:17:53 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:17:53] "GET /metrics HTTP/1.1" 200 38179 "" "Prometheus/2.43.0" 2024-04-02T05:17:53.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:53 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:17:53.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:53 smithi086 ceph-mon[88839]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:17:53.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:53 smithi086 ceph-mon[88839]: osd.3 marked itself down and dead 2024-04-02T05:17:53.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:53 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:17:53.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:53 smithi086 ceph-mon[86443]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:17:53.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:53 smithi086 ceph-mon[86443]: osd.3 marked itself down and dead 2024-04-02T05:17:54.136 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:53 smithi086 podman[113318]: 2024-04-02 05:17:53.838984419 +0000 UTC m=+1.027829525 container died 327538dac44e725173e5349cdca9739ed237e16f2acdf509a894dad7500905b8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, io.openshift.expose-services=, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, vcs-type=git, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, ceph=True) 2024-04-02T05:17:54.387 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:54 smithi086 podman[113318]: 2024-04-02 05:17:54.366300965 +0000 UTC m=+1.555146076 container cleanup 327538dac44e725173e5349cdca9739ed237e16f2acdf509a894dad7500905b8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, release=754, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, name=centos-stream, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-02T05:17:54.387 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:54 smithi086 bash[113318]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3 2024-04-02T05:17:54.387 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:54 smithi086 ceph-mon[86443]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-02T05:17:54.387 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:54 smithi086 ceph-mon[86443]: failed to encode map e133 with expected crc 2024-04-02T05:17:54.387 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:54 smithi086 ceph-mon[86443]: failed to encode map e133 with expected crc 2024-04-02T05:17:54.387 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:54 smithi086 ceph-mon[86443]: osdmap e133: 8 total, 7 up, 8 in 2024-04-02T05:17:54.387 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:54 smithi086 ceph-mon[86443]: failed to encode map e133 with expected crc 2024-04-02T05:17:54.388 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:54 smithi086 ceph-mon[86443]: failed to encode map e133 with expected crc 2024-04-02T05:17:54.388 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:54 smithi086 ceph-mon[86443]: failed to encode map e133 with expected crc 2024-04-02T05:17:54.388 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:54 smithi086 ceph-mon[86443]: failed to encode map e133 with expected crc 2024-04-02T05:17:54.388 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:54 smithi086 ceph-mon[86443]: failed to encode map e133 with expected crc 2024-04-02T05:17:54.388 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:54 smithi086 ceph-mon[86443]: failed to encode map e133 with expected crc 2024-04-02T05:17:54.388 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:54 smithi086 ceph-mon[88839]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-02T05:17:54.388 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:54 smithi086 ceph-mon[88839]: failed to encode map e133 with expected crc 2024-04-02T05:17:54.388 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:54 smithi086 ceph-mon[88839]: failed to encode map e133 with expected crc 2024-04-02T05:17:54.388 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:54 smithi086 ceph-mon[88839]: osdmap e133: 8 total, 7 up, 8 in 2024-04-02T05:17:54.388 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:54 smithi086 ceph-mon[88839]: failed to encode map e133 with expected crc 2024-04-02T05:17:54.388 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:54 smithi086 ceph-mon[88839]: failed to encode map e133 with expected crc 2024-04-02T05:17:54.388 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:54 smithi086 ceph-mon[88839]: failed to encode map e133 with expected crc 2024-04-02T05:17:54.389 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:54 smithi086 ceph-mon[88839]: failed to encode map e133 with expected crc 2024-04-02T05:17:54.389 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:54 smithi086 ceph-mon[88839]: failed to encode map e133 with expected crc 2024-04-02T05:17:54.389 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:54 smithi086 ceph-mon[88839]: failed to encode map e133 with expected crc 2024-04-02T05:17:54.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:54 smithi192 ceph-mon[68000]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-02T05:17:54.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:54 smithi192 ceph-mon[68000]: failed to encode map e133 with expected crc 2024-04-02T05:17:54.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:54 smithi192 ceph-mon[68000]: failed to encode map e133 with expected crc 2024-04-02T05:17:54.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:54 smithi192 ceph-mon[68000]: osdmap e133: 8 total, 7 up, 8 in 2024-04-02T05:17:54.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:54 smithi192 ceph-mon[68000]: failed to encode map e133 with expected crc 2024-04-02T05:17:54.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:54 smithi192 ceph-mon[68000]: failed to encode map e133 with expected crc 2024-04-02T05:17:54.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:54 smithi192 ceph-mon[68000]: failed to encode map e133 with expected crc 2024-04-02T05:17:54.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:54 smithi192 ceph-mon[68000]: failed to encode map e133 with expected crc 2024-04-02T05:17:54.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:54 smithi192 ceph-mon[68000]: failed to encode map e133 with expected crc 2024-04-02T05:17:54.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:54 smithi192 ceph-mon[68000]: failed to encode map e133 with expected crc 2024-04-02T05:17:54.900 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:54 smithi086 podman[113336]: 2024-04-02 05:17:54.617933595 +0000 UTC m=+0.781905815 container remove 327538dac44e725173e5349cdca9739ed237e16f2acdf509a894dad7500905b8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, name=centos-stream, io.openshift.expose-services=, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, distribution-scope=public, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git) 2024-04-02T05:17:55.181 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:55 smithi086 podman[113434]: 2024-04-02 05:17:54.969552205 +0000 UTC m=+0.017981911 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:17:55.181 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:55 smithi086 podman[113434]: 2024-04-02 05:17:55.077168508 +0000 UTC m=+0.125598213 container create f88af57cf27c01c79fa4a0b53969a796a45eded99d6f1579402eef312d780921 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3-deactivate, org.label-schema.build-date=20240326, RELEASE=squid-6cec793, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD) 2024-04-02T05:17:55.583 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:55 smithi086 podman[113434]: 2024-04-02 05:17:55.296345299 +0000 UTC m=+0.344775004 container init f88af57cf27c01c79fa4a0b53969a796a45eded99d6f1579402eef312d780921 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-6cec793, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=) 2024-04-02T05:17:55.583 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:55 smithi086 podman[113434]: 2024-04-02 05:17:55.299366348 +0000 UTC m=+0.347796064 container start f88af57cf27c01c79fa4a0b53969a796a45eded99d6f1579402eef312d780921 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, RELEASE=squid-6cec793, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-04-02T05:17:55.583 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:55 smithi086 podman[113434]: 2024-04-02 05:17:55.396597926 +0000 UTC m=+0.445027645 container attach f88af57cf27c01c79fa4a0b53969a796a45eded99d6f1579402eef312d780921 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3-deactivate, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-6cec793, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-02T05:17:55.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:55 smithi086 ceph-mon[88839]: pgmap v137: 161 pgs: 7 peering, 25 stale+active+clean, 129 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:17:55.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:55 smithi086 ceph-mon[88839]: failed to encode map e134 with expected crc 2024-04-02T05:17:55.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:55 smithi086 ceph-mon[88839]: osdmap e134: 8 total, 7 up, 8 in 2024-04-02T05:17:55.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:55 smithi086 ceph-mon[88839]: failed to encode map e134 with expected crc 2024-04-02T05:17:55.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:55 smithi086 ceph-mon[88839]: failed to encode map e134 with expected crc 2024-04-02T05:17:55.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:55 smithi086 ceph-mon[88839]: failed to encode map e134 with expected crc 2024-04-02T05:17:55.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:55 smithi086 ceph-mon[88839]: failed to encode map e134 with expected crc 2024-04-02T05:17:55.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:55 smithi086 ceph-mon[88839]: failed to encode map e134 with expected crc 2024-04-02T05:17:55.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:55 smithi086 ceph-mon[88839]: failed to encode map e134 with expected crc 2024-04-02T05:17:55.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:55 smithi086 ceph-mon[88839]: failed to encode map e134 with expected crc 2024-04-02T05:17:55.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:55 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:55.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:55 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:17:55.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:55 smithi086 ceph-mon[86443]: pgmap v137: 161 pgs: 7 peering, 25 stale+active+clean, 129 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:17:55.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:55 smithi086 ceph-mon[86443]: failed to encode map e134 with expected crc 2024-04-02T05:17:55.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:55 smithi086 ceph-mon[86443]: osdmap e134: 8 total, 7 up, 8 in 2024-04-02T05:17:55.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:55 smithi086 ceph-mon[86443]: failed to encode map e134 with expected crc 2024-04-02T05:17:55.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:55 smithi086 ceph-mon[86443]: failed to encode map e134 with expected crc 2024-04-02T05:17:55.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:55 smithi086 ceph-mon[86443]: failed to encode map e134 with expected crc 2024-04-02T05:17:55.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:55 smithi086 ceph-mon[86443]: failed to encode map e134 with expected crc 2024-04-02T05:17:55.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:55 smithi086 ceph-mon[86443]: failed to encode map e134 with expected crc 2024-04-02T05:17:55.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:55 smithi086 ceph-mon[86443]: failed to encode map e134 with expected crc 2024-04-02T05:17:55.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:55 smithi086 ceph-mon[86443]: failed to encode map e134 with expected crc 2024-04-02T05:17:55.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:55 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:55.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:55 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:17:55.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:55 smithi192 ceph-mon[68000]: pgmap v137: 161 pgs: 7 peering, 25 stale+active+clean, 129 active+clean; 585 KiB data, 158 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:17:55.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:55 smithi192 ceph-mon[68000]: failed to encode map e134 with expected crc 2024-04-02T05:17:55.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:55 smithi192 ceph-mon[68000]: osdmap e134: 8 total, 7 up, 8 in 2024-04-02T05:17:55.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:55 smithi192 ceph-mon[68000]: failed to encode map e134 with expected crc 2024-04-02T05:17:55.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:55 smithi192 ceph-mon[68000]: failed to encode map e134 with expected crc 2024-04-02T05:17:55.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:55 smithi192 ceph-mon[68000]: failed to encode map e134 with expected crc 2024-04-02T05:17:55.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:55 smithi192 ceph-mon[68000]: failed to encode map e134 with expected crc 2024-04-02T05:17:55.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:55 smithi192 ceph-mon[68000]: failed to encode map e134 with expected crc 2024-04-02T05:17:55.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:55 smithi192 ceph-mon[68000]: failed to encode map e134 with expected crc 2024-04-02T05:17:55.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:55 smithi192 ceph-mon[68000]: failed to encode map e134 with expected crc 2024-04-02T05:17:55.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:55 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:55.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:55 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:17:56.082 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:55 smithi086 podman[113434]: 2024-04-02 05:17:55.591521125 +0000 UTC m=+0.639950835 container died f88af57cf27c01c79fa4a0b53969a796a45eded99d6f1579402eef312d780921 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-6cec793, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-02T05:17:56.333 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:56 smithi086 podman[113460]: 2024-04-02 05:17:56.266977991 +0000 UTC m=+0.670421920 container remove f88af57cf27c01c79fa4a0b53969a796a45eded99d6f1579402eef312d780921 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , ceph=True, RELEASE=squid-6cec793, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-04-02T05:17:56.333 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:56 smithi086 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.3.service: Deactivated successfully. 2024-04-02T05:17:56.333 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:56 smithi086 systemd[1]: Stopped Ceph osd.3 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:17:56.333 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:56 smithi086 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.3.service: Consumed 18.239s CPU time. 2024-04-02T05:17:56.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:56 smithi192 ceph-mon[68000]: pgmap v139: 161 pgs: 12 active+undersized+degraded, 7 peering, 11 stale+active+clean, 19 active+undersized, 112 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 37/723 objects degraded (5.118%) 2024-04-02T05:17:56.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:56 smithi192 ceph-mon[68000]: Health check failed: Degraded data redundancy: 37/723 objects degraded (5.118%), 12 pgs degraded (PG_DEGRADED) 2024-04-02T05:17:56.718 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:56 smithi086 systemd[1]: Starting Ceph osd.3 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:17:56.718 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:56 smithi086 ceph-mon[86443]: pgmap v139: 161 pgs: 12 active+undersized+degraded, 7 peering, 11 stale+active+clean, 19 active+undersized, 112 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 37/723 objects degraded (5.118%) 2024-04-02T05:17:56.718 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:56 smithi086 ceph-mon[86443]: Health check failed: Degraded data redundancy: 37/723 objects degraded (5.118%), 12 pgs degraded (PG_DEGRADED) 2024-04-02T05:17:56.718 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:56 smithi086 ceph-mon[88839]: pgmap v139: 161 pgs: 12 active+undersized+degraded, 7 peering, 11 stale+active+clean, 19 active+undersized, 112 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 37/723 objects degraded (5.118%) 2024-04-02T05:17:56.718 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:56 smithi086 ceph-mon[88839]: Health check failed: Degraded data redundancy: 37/723 objects degraded (5.118%), 12 pgs degraded (PG_DEGRADED) 2024-04-02T05:17:56.973 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:56 smithi086 podman[113567]: 2024-04-02 05:17:56.618937023 +0000 UTC m=+0.018211898 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:17:56.973 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:56 smithi086 podman[113567]: 2024-04-02 05:17:56.752799048 +0000 UTC m=+0.152073912 container create ae896489035e25737074e89212b597bb0890c76bff050db1d3def1387878eb86 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3-activate, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=squid-6cec793, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-02T05:17:57.333 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:56 smithi086 podman[113567]: 2024-04-02 05:17:56.970804682 +0000 UTC m=+0.370079549 container init ae896489035e25737074e89212b597bb0890c76bff050db1d3def1387878eb86 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3-activate, RELEASE=squid-6cec793, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240326, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-04-02T05:17:57.333 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:56 smithi086 podman[113567]: 2024-04-02 05:17:56.974290618 +0000 UTC m=+0.373565490 container start ae896489035e25737074e89212b597bb0890c76bff050db1d3def1387878eb86 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3-activate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, RELEASE=squid-6cec793, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-04-02T05:17:57.333 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:57 smithi086 podman[113567]: 2024-04-02 05:17:57.062773446 +0000 UTC m=+0.462048317 container attach ae896489035e25737074e89212b597bb0890c76bff050db1d3def1387878eb86 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3-activate, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, RELEASE=squid-6cec793, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-04-02T05:17:57.727 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:57 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3-activate[113582]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-02T05:17:57.727 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:57 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3-activate[113582]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-02T05:17:57.727 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:57 smithi086 bash[113567]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-02T05:17:57.727 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:57 smithi086 bash[113567]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-02T05:17:57.727 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:57 smithi086 bash[113567]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-02T05:17:57.727 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:57 smithi086 bash[113567]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-02T05:17:57.727 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:57 smithi086 bash[113567]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-02T05:17:57.727 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:57 smithi086 bash[113567]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-02T05:17:57.727 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:57 smithi086 bash[113567]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-02T05:17:57.727 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:57 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3-activate[113582]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-02T05:17:57.728 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:57 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3-activate[113582]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-02T05:17:57.728 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:57 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3-activate[113582]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-02T05:17:57.728 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:57 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3-activate[113582]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-02T05:17:57.728 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:57 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3-activate[113582]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-02T05:17:57.728 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:57 smithi086 podman[113567]: 2024-04-02 05:17:57.436923556 +0000 UTC m=+0.836198421 container died ae896489035e25737074e89212b597bb0890c76bff050db1d3def1387878eb86 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3-activate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=squid-6cec793, org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-04-02T05:17:58.466 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:58 smithi086 podman[113702]: 2024-04-02 05:17:58.116694345 +0000 UTC m=+0.675009971 container remove ae896489035e25737074e89212b597bb0890c76bff050db1d3def1387878eb86 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3-activate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-6cec793, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-02T05:17:58.467 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:58 smithi086 podman[113764]: 2024-04-02 05:17:58.2364185 +0000 UTC m=+0.018131953 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:17:58.467 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:58 smithi086 podman[113764]: 2024-04-02 05:17:58.3508197 +0000 UTC m=+0.132533152 container create 6028f927dfd6758fd491d8de4f84bb5e7fd53eb3a3a4c3a2fe7f464891282bc9 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, RELEASE=squid-6cec793, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=) 2024-04-02T05:17:58.833 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:58 smithi086 podman[113764]: 2024-04-02 05:17:58.562002632 +0000 UTC m=+0.343716088 container init 6028f927dfd6758fd491d8de4f84bb5e7fd53eb3a3a4c3a2fe7f464891282bc9 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, RELEASE=squid-6cec793, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-04-02T05:17:58.833 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:58 smithi086 podman[113764]: 2024-04-02 05:17:58.565159106 +0000 UTC m=+0.346872563 container start 6028f927dfd6758fd491d8de4f84bb5e7fd53eb3a3a4c3a2fe7f464891282bc9 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3, org.label-schema.build-date=20240326, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-6cec793, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-04-02T05:17:58.833 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:58 smithi086 ceph-osd[113783]: -- 172.21.15.86:0/4040828368 <== mon.0 v2:172.21.15.86:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d449e73680 con 0x55d449e3d400 2024-04-02T05:17:58.833 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:58 smithi086 bash[113764]: 6028f927dfd6758fd491d8de4f84bb5e7fd53eb3a3a4c3a2fe7f464891282bc9 2024-04-02T05:17:58.833 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:58 smithi086 systemd[1]: Started Ceph osd.3 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:17:59.519 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:17:59 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3[113778]: 2024-04-02T05:17:59.112+0000 7f2e5920b7c0 -1 Falling back to public interface 2024-04-02T05:17:59.519 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:59 smithi086 ceph-mon[88839]: pgmap v140: 161 pgs: 21 active+undersized+degraded, 7 peering, 37 active+undersized, 96 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 87/723 objects degraded (12.033%) 2024-04-02T05:17:59.519 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:59 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:59.519 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:17:59 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:59.520 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:59 smithi086 ceph-mon[86443]: pgmap v140: 161 pgs: 21 active+undersized+degraded, 7 peering, 37 active+undersized, 96 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 87/723 objects degraded (12.033%) 2024-04-02T05:17:59.520 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:59 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:59.520 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:17:59 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:59.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:59 smithi192 ceph-mon[68000]: pgmap v140: 161 pgs: 21 active+undersized+degraded, 7 peering, 37 active+undersized, 96 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 87/723 objects degraded (12.033%) 2024-04-02T05:17:59.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:59 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:17:59.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:17:59 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:01.481 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:01 smithi086 ceph-mon[86443]: pgmap v141: 161 pgs: 23 active+undersized+degraded, 42 active+undersized, 96 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 89/723 objects degraded (12.310%) 2024-04-02T05:18:01.481 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:01 smithi086 ceph-mon[88839]: pgmap v141: 161 pgs: 23 active+undersized+degraded, 42 active+undersized, 96 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 89/723 objects degraded (12.310%) 2024-04-02T05:18:01.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:01 smithi192 ceph-mon[68000]: pgmap v141: 161 pgs: 23 active+undersized+degraded, 42 active+undersized, 96 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 89/723 objects degraded (12.310%) 2024-04-02T05:18:02.083 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:18:01 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3[113778]: 2024-04-02T05:18:01.821+0000 7f2e5920b7c0 -1 osd.3 0 read_superblock omap replica is missing. 2024-04-02T05:18:02.083 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:18:01 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3[113778]: 2024-04-02T05:18:01.854+0000 7f2e5920b7c0 -1 osd.3 132 log_to_monitors true 2024-04-02T05:18:02.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:02 smithi192 ceph-mon[68000]: from='osd.3 [v2:172.21.15.86:6826/1820604723,v1:172.21.15.86:6827/1820604723]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-02T05:18:02.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:02 smithi192 ceph-mon[68000]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-02T05:18:02.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:02 smithi192 ceph-mon[68000]: Health check update: Degraded data redundancy: 89/723 objects degraded (12.310%), 23 pgs degraded (PG_DEGRADED) 2024-04-02T05:18:02.696 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:18:02 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3[113778]: 2024-04-02T05:18:02.384+0000 7f2e4a663700 -1 osd.3 132 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T05:18:02.697 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:02 smithi086 ceph-mon[86443]: from='osd.3 [v2:172.21.15.86:6826/1820604723,v1:172.21.15.86:6827/1820604723]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-02T05:18:02.697 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:02 smithi086 ceph-mon[86443]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-02T05:18:02.697 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:02 smithi086 ceph-mon[86443]: Health check update: Degraded data redundancy: 89/723 objects degraded (12.310%), 23 pgs degraded (PG_DEGRADED) 2024-04-02T05:18:02.697 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:02 smithi086 ceph-mon[88839]: from='osd.3 [v2:172.21.15.86:6826/1820604723,v1:172.21.15.86:6827/1820604723]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-02T05:18:02.697 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:02 smithi086 ceph-mon[88839]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-02T05:18:02.697 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:02 smithi086 ceph-mon[88839]: Health check update: Degraded data redundancy: 89/723 objects degraded (12.310%), 23 pgs degraded (PG_DEGRADED) 2024-04-02T05:18:03.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:03 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:18:03.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:03 smithi192 ceph-mon[68000]: pgmap v142: 161 pgs: 23 active+undersized+degraded, 42 active+undersized, 96 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 89/723 objects degraded (12.310%) 2024-04-02T05:18:03.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:03 smithi192 ceph-mon[68000]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-02T05:18:03.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:03 smithi192 ceph-mon[68000]: failed to encode map e135 with expected crc 2024-04-02T05:18:03.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:03 smithi192 ceph-mon[68000]: osdmap e135: 8 total, 7 up, 8 in 2024-04-02T05:18:03.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:03 smithi192 ceph-mon[68000]: from='osd.3 [v2:172.21.15.86:6826/1820604723,v1:172.21.15.86:6827/1820604723]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T05:18:03.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:03 smithi192 ceph-mon[68000]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T05:18:03.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:03 smithi192 ceph-mon[68000]: failed to encode map e135 with expected crc 2024-04-02T05:18:03.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:03 smithi192 ceph-mon[68000]: failed to encode map e135 with expected crc 2024-04-02T05:18:03.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:03 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:03.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:03 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:03.832 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:18:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:18:03] "GET /metrics HTTP/1.1" 200 38181 "" "Prometheus/2.43.0" 2024-04-02T05:18:03.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:03 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:18:03.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:03 smithi086 ceph-mon[88839]: pgmap v142: 161 pgs: 23 active+undersized+degraded, 42 active+undersized, 96 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 89/723 objects degraded (12.310%) 2024-04-02T05:18:03.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:03 smithi086 ceph-mon[88839]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-02T05:18:03.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:03 smithi086 ceph-mon[88839]: failed to encode map e135 with expected crc 2024-04-02T05:18:03.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:03 smithi086 ceph-mon[88839]: osdmap e135: 8 total, 7 up, 8 in 2024-04-02T05:18:03.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:03 smithi086 ceph-mon[88839]: from='osd.3 [v2:172.21.15.86:6826/1820604723,v1:172.21.15.86:6827/1820604723]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T05:18:03.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:03 smithi086 ceph-mon[88839]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T05:18:03.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:03 smithi086 ceph-mon[88839]: failed to encode map e135 with expected crc 2024-04-02T05:18:03.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:03 smithi086 ceph-mon[88839]: failed to encode map e135 with expected crc 2024-04-02T05:18:03.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:03 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:03.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:03 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:03.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:03 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:18:03.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:03 smithi086 ceph-mon[86443]: pgmap v142: 161 pgs: 23 active+undersized+degraded, 42 active+undersized, 96 active+clean; 585 KiB data, 159 MiB used, 715 GiB / 715 GiB avail; 89/723 objects degraded (12.310%) 2024-04-02T05:18:03.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:03 smithi086 ceph-mon[86443]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-02T05:18:03.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:03 smithi086 ceph-mon[86443]: failed to encode map e135 with expected crc 2024-04-02T05:18:03.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:03 smithi086 ceph-mon[86443]: osdmap e135: 8 total, 7 up, 8 in 2024-04-02T05:18:03.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:03 smithi086 ceph-mon[86443]: from='osd.3 [v2:172.21.15.86:6826/1820604723,v1:172.21.15.86:6827/1820604723]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T05:18:03.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:03 smithi086 ceph-mon[86443]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-02T05:18:03.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:03 smithi086 ceph-mon[86443]: failed to encode map e135 with expected crc 2024-04-02T05:18:03.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:03 smithi086 ceph-mon[86443]: failed to encode map e135 with expected crc 2024-04-02T05:18:03.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:03 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:03.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:03 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:04.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:04 smithi086 ceph-mon[88839]: failed to encode map e135 with expected crc 2024-04-02T05:18:04.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:04 smithi086 ceph-mon[88839]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-02T05:18:04.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:04 smithi086 ceph-mon[88839]: failed to encode map e136 with expected crc 2024-04-02T05:18:04.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:04 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T05:18:04.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:04 smithi086 ceph-mon[88839]: failed to encode map e136 with expected crc 2024-04-02T05:18:04.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:04 smithi086 ceph-mon[88839]: osd.3 [v2:172.21.15.86:6826/1820604723,v1:172.21.15.86:6827/1820604723] boot 2024-04-02T05:18:04.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:04 smithi086 ceph-mon[88839]: failed to encode map e136 with expected crc 2024-04-02T05:18:04.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:04 smithi086 ceph-mon[88839]: osdmap e136: 8 total, 8 up, 8 in 2024-04-02T05:18:04.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:04 smithi086 ceph-mon[88839]: failed to encode map e136 with expected crc 2024-04-02T05:18:04.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:04 smithi086 ceph-mon[88839]: failed to encode map e136 with expected crc 2024-04-02T05:18:04.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:04 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:04.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:04 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:04.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:04 smithi086 ceph-mon[88839]: pgmap v145: 161 pgs: 5 peering, 22 active+undersized+degraded, 38 active+undersized, 96 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 88/723 objects degraded (12.172%) 2024-04-02T05:18:04.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:04 smithi086 ceph-mon[88839]: osdmap e137: 8 total, 8 up, 8 in 2024-04-02T05:18:04.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:04 smithi086 ceph-mon[86443]: failed to encode map e135 with expected crc 2024-04-02T05:18:04.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:04 smithi086 ceph-mon[86443]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-02T05:18:04.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:04 smithi086 ceph-mon[86443]: failed to encode map e136 with expected crc 2024-04-02T05:18:04.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:04 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T05:18:04.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:04 smithi086 ceph-mon[86443]: failed to encode map e136 with expected crc 2024-04-02T05:18:04.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:04 smithi086 ceph-mon[86443]: osd.3 [v2:172.21.15.86:6826/1820604723,v1:172.21.15.86:6827/1820604723] boot 2024-04-02T05:18:04.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:04 smithi086 ceph-mon[86443]: failed to encode map e136 with expected crc 2024-04-02T05:18:04.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:04 smithi086 ceph-mon[86443]: osdmap e136: 8 total, 8 up, 8 in 2024-04-02T05:18:04.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:04 smithi086 ceph-mon[86443]: failed to encode map e136 with expected crc 2024-04-02T05:18:04.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:04 smithi086 ceph-mon[86443]: failed to encode map e136 with expected crc 2024-04-02T05:18:04.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:04 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:04.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:04 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:04.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:04 smithi086 ceph-mon[86443]: pgmap v145: 161 pgs: 5 peering, 22 active+undersized+degraded, 38 active+undersized, 96 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 88/723 objects degraded (12.172%) 2024-04-02T05:18:04.585 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:04 smithi086 ceph-mon[86443]: osdmap e137: 8 total, 8 up, 8 in 2024-04-02T05:18:04.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:04 smithi192 ceph-mon[68000]: failed to encode map e135 with expected crc 2024-04-02T05:18:04.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:04 smithi192 ceph-mon[68000]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-02T05:18:04.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:04 smithi192 ceph-mon[68000]: failed to encode map e136 with expected crc 2024-04-02T05:18:04.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:04 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T05:18:04.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:04 smithi192 ceph-mon[68000]: failed to encode map e136 with expected crc 2024-04-02T05:18:04.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:04 smithi192 ceph-mon[68000]: osd.3 [v2:172.21.15.86:6826/1820604723,v1:172.21.15.86:6827/1820604723] boot 2024-04-02T05:18:04.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:04 smithi192 ceph-mon[68000]: failed to encode map e136 with expected crc 2024-04-02T05:18:04.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:04 smithi192 ceph-mon[68000]: osdmap e136: 8 total, 8 up, 8 in 2024-04-02T05:18:04.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:04 smithi192 ceph-mon[68000]: failed to encode map e136 with expected crc 2024-04-02T05:18:04.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:04 smithi192 ceph-mon[68000]: failed to encode map e136 with expected crc 2024-04-02T05:18:04.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:04 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:04.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:04 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:04.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:04 smithi192 ceph-mon[68000]: pgmap v145: 161 pgs: 5 peering, 22 active+undersized+degraded, 38 active+undersized, 96 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 88/723 objects degraded (12.172%) 2024-04-02T05:18:04.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:04 smithi192 ceph-mon[68000]: osdmap e137: 8 total, 8 up, 8 in 2024-04-02T05:18:05.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:05 smithi192 ceph-mon[68000]: failed to encode map e137 with expected crc 2024-04-02T05:18:05.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:05 smithi192 ceph-mon[68000]: failed to encode map e137 with expected crc 2024-04-02T05:18:05.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:05 smithi192 ceph-mon[68000]: failed to encode map e137 with expected crc 2024-04-02T05:18:05.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:05 smithi192 ceph-mon[68000]: failed to encode map e137 with expected crc 2024-04-02T05:18:05.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:05 smithi192 ceph-mon[68000]: failed to encode map e137 with expected crc 2024-04-02T05:18:05.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:05 smithi192 ceph-mon[68000]: failed to encode map e137 with expected crc 2024-04-02T05:18:05.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:05 smithi192 ceph-mon[68000]: failed to encode map e137 with expected crc 2024-04-02T05:18:05.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:05 smithi192 ceph-mon[68000]: failed to encode map e137 with expected crc 2024-04-02T05:18:05.745 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:05 smithi086 ceph-mon[88839]: failed to encode map e137 with expected crc 2024-04-02T05:18:05.746 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:05 smithi086 ceph-mon[88839]: failed to encode map e137 with expected crc 2024-04-02T05:18:05.746 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:05 smithi086 ceph-mon[88839]: failed to encode map e137 with expected crc 2024-04-02T05:18:05.746 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:05 smithi086 ceph-mon[88839]: failed to encode map e137 with expected crc 2024-04-02T05:18:05.746 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:05 smithi086 ceph-mon[88839]: failed to encode map e137 with expected crc 2024-04-02T05:18:05.746 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:05 smithi086 ceph-mon[88839]: failed to encode map e137 with expected crc 2024-04-02T05:18:05.746 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:05 smithi086 ceph-mon[88839]: failed to encode map e137 with expected crc 2024-04-02T05:18:05.746 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:05 smithi086 ceph-mon[88839]: failed to encode map e137 with expected crc 2024-04-02T05:18:05.746 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:05 smithi086 ceph-mon[86443]: failed to encode map e137 with expected crc 2024-04-02T05:18:05.746 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:05 smithi086 ceph-mon[86443]: failed to encode map e137 with expected crc 2024-04-02T05:18:05.747 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:05 smithi086 ceph-mon[86443]: failed to encode map e137 with expected crc 2024-04-02T05:18:05.747 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:05 smithi086 ceph-mon[86443]: failed to encode map e137 with expected crc 2024-04-02T05:18:05.747 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:05 smithi086 ceph-mon[86443]: failed to encode map e137 with expected crc 2024-04-02T05:18:05.747 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:05 smithi086 ceph-mon[86443]: failed to encode map e137 with expected crc 2024-04-02T05:18:05.747 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:05 smithi086 ceph-mon[86443]: failed to encode map e137 with expected crc 2024-04-02T05:18:05.747 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:05 smithi086 ceph-mon[86443]: failed to encode map e137 with expected crc 2024-04-02T05:18:06.453 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:06 smithi086 ceph-mon[88839]: pgmap v147: 161 pgs: 5 peering, 15 active+undersized+degraded, 28 active+undersized, 113 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 71/723 objects degraded (9.820%) 2024-04-02T05:18:06.453 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:06 smithi086 ceph-mon[86443]: pgmap v147: 161 pgs: 5 peering, 15 active+undersized+degraded, 28 active+undersized, 113 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 71/723 objects degraded (9.820%) 2024-04-02T05:18:06.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:06 smithi192 ceph-mon[68000]: pgmap v147: 161 pgs: 5 peering, 15 active+undersized+degraded, 28 active+undersized, 113 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 71/723 objects degraded (9.820%) 2024-04-02T05:18:07.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:07 smithi192 ceph-mon[68000]: Health check update: Degraded data redundancy: 71/723 objects degraded (9.820%), 15 pgs degraded (PG_DEGRADED) 2024-04-02T05:18:07.782 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:07 smithi086 ceph-mon[88839]: Health check update: Degraded data redundancy: 71/723 objects degraded (9.820%), 15 pgs degraded (PG_DEGRADED) 2024-04-02T05:18:07.783 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:07 smithi086 ceph-mon[86443]: Health check update: Degraded data redundancy: 71/723 objects degraded (9.820%), 15 pgs degraded (PG_DEGRADED) 2024-04-02T05:18:08.082 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:18:08 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:18:08.018+0000 7fe19d012700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-04-02T05:18:09.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:08 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:09.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:08 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:09.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:08 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:18:09.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:08 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:18:09.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:08 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:09.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:08 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:18:09.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:08 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:18:09.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:08 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:18:09.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:08 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:18:09.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:08 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-02T05:18:09.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:08 smithi192 ceph-mon[68000]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-02T05:18:09.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:08 smithi192 ceph-mon[68000]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-04-02T05:18:09.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:08 smithi192 ceph-mon[68000]: pgmap v148: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:18:09.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:08 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:09.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:08 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:09.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:08 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:18:09.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:08 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:18:09.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:08 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:09.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:08 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:18:09.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:08 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:18:09.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:08 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:18:09.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:08 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:18:09.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:08 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-02T05:18:09.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:08 smithi086 ceph-mon[88839]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-02T05:18:09.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:08 smithi086 ceph-mon[88839]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-04-02T05:18:09.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:08 smithi086 ceph-mon[88839]: pgmap v148: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:18:09.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:08 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:09.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:08 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:09.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:08 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:18:09.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:08 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:18:09.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:08 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:09.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:08 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:18:09.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:08 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:18:09.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:08 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:18:09.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:08 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:18:09.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:08 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-02T05:18:09.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:08 smithi086 ceph-mon[86443]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-02T05:18:09.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:08 smithi086 ceph-mon[86443]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-04-02T05:18:09.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:08 smithi086 ceph-mon[86443]: pgmap v148: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:18:10.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:09 smithi192 ceph-mon[68000]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 71/723 objects degraded (9.820%), 15 pgs degraded) 2024-04-02T05:18:10.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:09 smithi192 ceph-mon[68000]: Cluster is now healthy 2024-04-02T05:18:10.332 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:09 smithi086 ceph-mon[88839]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 71/723 objects degraded (9.820%), 15 pgs degraded) 2024-04-02T05:18:10.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:09 smithi086 ceph-mon[88839]: Cluster is now healthy 2024-04-02T05:18:10.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:09 smithi086 ceph-mon[86443]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 71/723 objects degraded (9.820%), 15 pgs degraded) 2024-04-02T05:18:10.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:09 smithi086 ceph-mon[86443]: Cluster is now healthy 2024-04-02T05:18:11.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:11 smithi086 ceph-mon[86443]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:18:11.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:11 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:11.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:11 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:18:11.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:11 smithi086 ceph-mon[88839]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:18:11.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:11 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:11.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:11 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:18:11.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:11 smithi192 ceph-mon[68000]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:18:11.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:11 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:11.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:11 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:18:13.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:13 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:18:13.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:13 smithi086 ceph-mon[86443]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:18:13.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:18:13 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:18:13] "GET /metrics HTTP/1.1" 200 38188 "" "Prometheus/2.43.0" 2024-04-02T05:18:13.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:13 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:18:13.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:13 smithi086 ceph-mon[88839]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:18:13.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:13 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:18:13.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:13 smithi192 ceph-mon[68000]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:18:15.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:15 smithi086 ceph-mon[88839]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-04-02T05:18:15.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:15 smithi086 ceph-mon[86443]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-04-02T05:18:15.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:15 smithi192 ceph-mon[68000]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-04-02T05:18:16.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:16 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:16.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:16 smithi192 ceph-mon[68000]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-04-02T05:18:16.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:16 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:16.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:16 smithi086 ceph-mon[86443]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-04-02T05:18:16.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:16 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:16.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:16 smithi086 ceph-mon[88839]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-04-02T05:18:19.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:19 smithi192 ceph-mon[68000]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-02T05:18:19.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:19 smithi086 ceph-mon[86443]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-02T05:18:19.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:19 smithi086 ceph-mon[88839]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-02T05:18:20.232 INFO:teuthology.orchestra.run.smithi086.stdout:true 2024-04-02T05:18:20.881 INFO:teuthology.orchestra.run.smithi086.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T05:18:20.881 INFO:teuthology.orchestra.run.smithi086.stdout:alertmanager.a smithi086 *:9093,9094 running (6m) 17s ago 28m 26.1M - 0.25.0 c8568f914cd2 660ea62bb6b9 2024-04-02T05:18:20.881 INFO:teuthology.orchestra.run.smithi086.stdout:grafana.a smithi192 *:3000 running (6m) 4m ago 28m 79.2M - 9.4.12 2bacad6d85d8 259af9e63aa5 2024-04-02T05:18:20.881 INFO:teuthology.orchestra.run.smithi086.stdout:iscsi.foo.smithi086.rsbqwx smithi086 running (5m) 17s ago 27m 62.1M - 3.5 e1d6a67b021e fe728eb940f7 2024-04-02T05:18:20.881 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.x smithi192 *:8443,9283,8765 running (7m) 4m ago 32m 449M - 19.0.0-1979-g6cec793d 78f830190839 dd1e6d0f0a95 2024-04-02T05:18:20.881 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.y smithi086 *:8443,9283,8765 running (17m) 17s ago 34m 507M - 19.0.0-1979-g6cec793d 78f830190839 3dca54a018ab 2024-04-02T05:18:20.881 INFO:teuthology.orchestra.run.smithi086.stdout:mon.a smithi086 running (4m) 17s ago 34m 51.7M 2048M 19.0.0-1979-g6cec793d 78f830190839 8e478dd4da2d 2024-04-02T05:18:20.882 INFO:teuthology.orchestra.run.smithi086.stdout:mon.b smithi192 running (5m) 4m ago 32m 36.2M 2048M 19.0.0-1979-g6cec793d 78f830190839 4125e997188f 2024-04-02T05:18:20.882 INFO:teuthology.orchestra.run.smithi086.stdout:mon.c smithi086 running (3m) 17s ago 33m 42.9M 2048M 19.0.0-1979-g6cec793d 78f830190839 8180999b3142 2024-04-02T05:18:20.882 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.a smithi086 *:9100 running (7m) 17s ago 29m 16.3M - 1.5.0 0da6a335fe13 05a844569d7b 2024-04-02T05:18:20.882 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.b smithi192 *:9100 running (7m) 4m ago 28m 17.4M - 1.5.0 0da6a335fe13 1a7d6788d58b 2024-04-02T05:18:20.882 INFO:teuthology.orchestra.run.smithi086.stdout:osd.0 smithi086 running (2m) 17s ago 32m 46.0M 2992M 19.0.0-1979-g6cec793d 78f830190839 e521a0ef7daf 2024-04-02T05:18:20.882 INFO:teuthology.orchestra.run.smithi086.stdout:osd.1 smithi086 running (2m) 17s ago 32m 53.1M 2992M 19.0.0-1979-g6cec793d 78f830190839 76fe72844771 2024-04-02T05:18:20.882 INFO:teuthology.orchestra.run.smithi086.stdout:osd.2 smithi086 running (76s) 17s ago 31m 48.9M 2992M 19.0.0-1979-g6cec793d 78f830190839 28594f089a8f 2024-04-02T05:18:20.882 INFO:teuthology.orchestra.run.smithi086.stdout:osd.3 smithi086 running (22s) 17s ago 31m 12.3M 2992M 19.0.0-1979-g6cec793d 78f830190839 6028f927dfd6 2024-04-02T05:18:20.882 INFO:teuthology.orchestra.run.smithi086.stdout:osd.4 smithi192 running (30m) 4m ago 30m 56.1M 3248M 17.2.0 e1d6a67b021e 6535d7919a2d 2024-04-02T05:18:20.882 INFO:teuthology.orchestra.run.smithi086.stdout:osd.5 smithi192 running (30m) 4m ago 30m 54.9M 3248M 17.2.0 e1d6a67b021e 7259249605f2 2024-04-02T05:18:20.882 INFO:teuthology.orchestra.run.smithi086.stdout:osd.6 smithi192 running (30m) 4m ago 30m 52.9M 3248M 17.2.0 e1d6a67b021e bb7d171b02a1 2024-04-02T05:18:20.882 INFO:teuthology.orchestra.run.smithi086.stdout:osd.7 smithi192 running (29m) 4m ago 29m 54.9M 3248M 17.2.0 e1d6a67b021e 95f5a0353999 2024-04-02T05:18:20.882 INFO:teuthology.orchestra.run.smithi086.stdout:prometheus.a smithi192 *:9095 running (6m) 4m ago 29m 65.1M - 2.43.0 a07b618ecd1d 8378fcdb60e3 2024-04-02T05:18:20.883 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi086.bxzxlp smithi086 *:8000 running (27m) 17s ago 27m 80.0M - 17.2.0 e1d6a67b021e 6725785fbc53 2024-04-02T05:18:20.883 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi192.pmlbot smithi192 *:8000 running (27m) 4m ago 27m 77.9M - 17.2.0 e1d6a67b021e 5ae3638f8b1c 2024-04-02T05:18:21.268 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:18:21.269 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": { 2024-04-02T05:18:21.269 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 3 2024-04-02T05:18:21.269 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:18:21.269 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": { 2024-04-02T05:18:21.269 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 2 2024-04-02T05:18:21.269 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:18:21.269 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": { 2024-04-02T05:18:21.269 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-04-02T05:18:21.269 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 4 2024-04-02T05:18:21.269 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:18:21.270 INFO:teuthology.orchestra.run.smithi086.stdout: "rgw": { 2024-04-02T05:18:21.270 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-02T05:18:21.270 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:18:21.270 INFO:teuthology.orchestra.run.smithi086.stdout: "overall": { 2024-04-02T05:18:21.270 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-04-02T05:18:21.270 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 9 2024-04-02T05:18:21.270 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-04-02T05:18:21.270 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:18:21.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:21 smithi086 ceph-mon[86443]: from='client.54441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:18:21.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:21 smithi086 ceph-mon[86443]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:18:21.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:21 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/719746620' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:18:21.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:21 smithi086 ceph-mon[88839]: from='client.54441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:18:21.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:21 smithi086 ceph-mon[88839]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:18:21.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:21 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/719746620' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:18:21.593 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:18:21.594 INFO:teuthology.orchestra.run.smithi086.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", 2024-04-02T05:18:21.594 INFO:teuthology.orchestra.run.smithi086.stdout: "in_progress": true, 2024-04-02T05:18:21.594 INFO:teuthology.orchestra.run.smithi086.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-04-02T05:18:21.594 INFO:teuthology.orchestra.run.smithi086.stdout: "services_complete": [], 2024-04-02T05:18:21.594 INFO:teuthology.orchestra.run.smithi086.stdout: "progress": "4/8 daemons upgraded", 2024-04-02T05:18:21.594 INFO:teuthology.orchestra.run.smithi086.stdout: "message": "Currently upgrading osd daemons", 2024-04-02T05:18:21.594 INFO:teuthology.orchestra.run.smithi086.stdout: "is_paused": false 2024-04-02T05:18:21.594 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:18:21.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:21 smithi192 ceph-mon[68000]: from='client.54441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:18:21.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:21 smithi192 ceph-mon[68000]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:18:21.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:21 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/719746620' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:18:22.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:22 smithi192 ceph-mon[68000]: from='client.54447 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:18:22.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:22 smithi192 ceph-mon[68000]: from='client.64396 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:18:22.832 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:22 smithi086 ceph-mon[86443]: from='client.54447 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:18:22.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:22 smithi086 ceph-mon[86443]: from='client.64396 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:18:22.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:22 smithi086 ceph-mon[88839]: from='client.54447 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:18:22.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:22 smithi086 ceph-mon[88839]: from='client.64396 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:18:23.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:23 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:18:23.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:23 smithi192 ceph-mon[68000]: from='client.54465 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:18:23.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:23 smithi192 ceph-mon[68000]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:18:23.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:23 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-02T05:18:23.735 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:18:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:18:23] "GET /metrics HTTP/1.1" 200 38188 "" "Prometheus/2.43.0" 2024-04-02T05:18:23.735 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:23 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:18:23.736 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:23 smithi086 ceph-mon[88839]: from='client.54465 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:18:23.736 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:23 smithi086 ceph-mon[88839]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:18:23.736 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:23 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-02T05:18:23.736 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:23 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:18:23.736 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:23 smithi086 ceph-mon[86443]: from='client.54465 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:18:23.737 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:23 smithi086 ceph-mon[86443]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:18:23.737 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:23 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-02T05:18:24.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:24 smithi192 ceph-mon[68000]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-02T05:18:24.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:24 smithi192 ceph-mon[68000]: Upgrade: osd.4 is safe to restart 2024-04-02T05:18:24.750 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:24 smithi086 ceph-mon[86443]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-02T05:18:24.750 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:24 smithi086 ceph-mon[86443]: Upgrade: osd.4 is safe to restart 2024-04-02T05:18:24.751 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:24 smithi086 ceph-mon[88839]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-02T05:18:24.751 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:24 smithi086 ceph-mon[88839]: Upgrade: osd.4 is safe to restart 2024-04-02T05:18:25.534 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:25 smithi192 ceph-mon[68000]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:18:25.535 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:25 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:25.535 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:25 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-02T05:18:25.535 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:25 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:18:25.535 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:25 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:18:25.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:25 smithi086 ceph-mon[86443]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:18:25.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:25 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:25.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:25 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-02T05:18:25.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:25 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:18:25.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:25 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:18:25.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:25 smithi086 ceph-mon[88839]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:18:25.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:25 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:25.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:25 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-02T05:18:25.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:25 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:18:25.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:25 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:18:26.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:26 smithi192 ceph-mon[68000]: Upgrade: Updating osd.4 2024-04-02T05:18:26.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:26 smithi192 ceph-mon[68000]: Deploying daemon osd.4 on smithi192 2024-04-02T05:18:26.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:26 smithi192 ceph-mon[68000]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:18:26.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:26 smithi086 ceph-mon[88839]: Upgrade: Updating osd.4 2024-04-02T05:18:26.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:26 smithi086 ceph-mon[88839]: Deploying daemon osd.4 on smithi192 2024-04-02T05:18:26.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:26 smithi086 ceph-mon[88839]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:18:26.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:26 smithi086 ceph-mon[86443]: Upgrade: Updating osd.4 2024-04-02T05:18:26.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:26 smithi086 ceph-mon[86443]: Deploying daemon osd.4 on smithi192 2024-04-02T05:18:26.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:26 smithi086 ceph-mon[86443]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:18:28.056 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:27 smithi192 systemd[1]: Stopping Ceph osd.4 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:18:28.056 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:27 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4[29147]: 2024-04-02T05:18:27.988+0000 7f9b19cf0700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T05:18:28.056 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:27 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4[29147]: 2024-04-02T05:18:27.988+0000 7f9b19cf0700 -1 osd.4 137 *** Got signal Terminated *** 2024-04-02T05:18:28.056 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:27 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4[29147]: 2024-04-02T05:18:27.988+0000 7f9b19cf0700 -1 osd.4 137 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T05:18:28.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:28 smithi086 ceph-mon[86443]: osd.4 marked itself down and dead 2024-04-02T05:18:28.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:28 smithi086 ceph-mon[88839]: osd.4 marked itself down and dead 2024-04-02T05:18:28.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:28 smithi192 ceph-mon[68000]: osd.4 marked itself down and dead 2024-04-02T05:18:28.782 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:28 smithi192 podman[77700]: 2024-04-02 05:18:28.461569224 +0000 UTC m=+0.568112353 container died 6535d7919a2d0f49eef3d9e49a417c5b99f2b6e0857bb71dfa7652272230f53b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4, name=centos-stream, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, version=8, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, io.openshift.expose-services=, ceph=True, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, architecture=x86_64, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870) 2024-04-02T05:18:29.071 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:28 smithi192 podman[77700]: 2024-04-02 05:18:28.780152663 +0000 UTC m=+0.886695800 container cleanup 6535d7919a2d0f49eef3d9e49a417c5b99f2b6e0857bb71dfa7652272230f53b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4, release=754, architecture=x86_64, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, build-date=2022-05-03T08:36:31.336870, name=centos-stream, distribution-scope=public, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, GIT_CLEAN=True) 2024-04-02T05:18:29.071 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:28 smithi192 bash[77700]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4 2024-04-02T05:18:29.071 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:28 smithi192 podman[77727]: 2024-04-02 05:18:28.990180757 +0000 UTC m=+0.527583946 container remove 6535d7919a2d0f49eef3d9e49a417c5b99f2b6e0857bb71dfa7652272230f53b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4, RELEASE=HEAD, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, architecture=x86_64, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public) 2024-04-02T05:18:29.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:29 smithi086 ceph-mon[86443]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-02T05:18:29.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:29 smithi086 ceph-mon[86443]: failed to encode map e138 with expected crc 2024-04-02T05:18:29.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:29 smithi086 ceph-mon[86443]: osdmap e138: 8 total, 7 up, 8 in 2024-04-02T05:18:29.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:29 smithi086 ceph-mon[86443]: failed to encode map e138 with expected crc 2024-04-02T05:18:29.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:29 smithi086 ceph-mon[86443]: failed to encode map e138 with expected crc 2024-04-02T05:18:29.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:29 smithi086 ceph-mon[86443]: failed to encode map e138 with expected crc 2024-04-02T05:18:29.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:29 smithi086 ceph-mon[86443]: failed to encode map e138 with expected crc 2024-04-02T05:18:29.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:29 smithi086 ceph-mon[86443]: failed to encode map e138 with expected crc 2024-04-02T05:18:29.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:29 smithi086 ceph-mon[86443]: failed to encode map e138 with expected crc 2024-04-02T05:18:29.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:29 smithi086 ceph-mon[86443]: failed to encode map e138 with expected crc 2024-04-02T05:18:29.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:29 smithi086 ceph-mon[86443]: failed to encode map e138 with expected crc 2024-04-02T05:18:29.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:29 smithi086 ceph-mon[86443]: pgmap v159: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:18:29.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:29 smithi086 ceph-mon[86443]: osdmap e139: 8 total, 7 up, 8 in 2024-04-02T05:18:29.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:29 smithi086 ceph-mon[88839]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-02T05:18:29.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:29 smithi086 ceph-mon[88839]: failed to encode map e138 with expected crc 2024-04-02T05:18:29.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:29 smithi086 ceph-mon[88839]: osdmap e138: 8 total, 7 up, 8 in 2024-04-02T05:18:29.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:29 smithi086 ceph-mon[88839]: failed to encode map e138 with expected crc 2024-04-02T05:18:29.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:29 smithi086 ceph-mon[88839]: failed to encode map e138 with expected crc 2024-04-02T05:18:29.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:29 smithi086 ceph-mon[88839]: failed to encode map e138 with expected crc 2024-04-02T05:18:29.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:29 smithi086 ceph-mon[88839]: failed to encode map e138 with expected crc 2024-04-02T05:18:29.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:29 smithi086 ceph-mon[88839]: failed to encode map e138 with expected crc 2024-04-02T05:18:29.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:29 smithi086 ceph-mon[88839]: failed to encode map e138 with expected crc 2024-04-02T05:18:29.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:29 smithi086 ceph-mon[88839]: failed to encode map e138 with expected crc 2024-04-02T05:18:29.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:29 smithi086 ceph-mon[88839]: failed to encode map e138 with expected crc 2024-04-02T05:18:29.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:29 smithi086 ceph-mon[88839]: pgmap v159: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:18:29.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:29 smithi086 ceph-mon[88839]: osdmap e139: 8 total, 7 up, 8 in 2024-04-02T05:18:29.405 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:29 smithi192 ceph-mon[68000]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-02T05:18:29.405 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:29 smithi192 ceph-mon[68000]: failed to encode map e138 with expected crc 2024-04-02T05:18:29.405 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:29 smithi192 ceph-mon[68000]: osdmap e138: 8 total, 7 up, 8 in 2024-04-02T05:18:29.405 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:29 smithi192 ceph-mon[68000]: failed to encode map e138 with expected crc 2024-04-02T05:18:29.405 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:29 smithi192 ceph-mon[68000]: failed to encode map e138 with expected crc 2024-04-02T05:18:29.405 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:29 smithi192 ceph-mon[68000]: failed to encode map e138 with expected crc 2024-04-02T05:18:29.405 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:29 smithi192 ceph-mon[68000]: failed to encode map e138 with expected crc 2024-04-02T05:18:29.405 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:29 smithi192 ceph-mon[68000]: failed to encode map e138 with expected crc 2024-04-02T05:18:29.405 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:29 smithi192 ceph-mon[68000]: failed to encode map e138 with expected crc 2024-04-02T05:18:29.405 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:29 smithi192 ceph-mon[68000]: failed to encode map e138 with expected crc 2024-04-02T05:18:29.405 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:29 smithi192 ceph-mon[68000]: failed to encode map e138 with expected crc 2024-04-02T05:18:29.406 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:29 smithi192 ceph-mon[68000]: pgmap v159: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:18:29.406 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:29 smithi192 ceph-mon[68000]: osdmap e139: 8 total, 7 up, 8 in 2024-04-02T05:18:29.662 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:29 smithi192 podman[77820]: 2024-04-02 05:18:29.305962748 +0000 UTC m=+0.018092958 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:18:29.663 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:29 smithi192 podman[77820]: 2024-04-02 05:18:29.433293198 +0000 UTC m=+0.145423443 container create f8d485e98507881d99fc71615f91415ecb91a34fdb8d29924fdb9e250e1ff428 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4-deactivate, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , RELEASE=squid-6cec793, GIT_CLEAN=True, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-02T05:18:29.663 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:29 smithi192 podman[77820]: 2024-04-02 05:18:29.660667311 +0000 UTC m=+0.372797511 container init f8d485e98507881d99fc71615f91415ecb91a34fdb8d29924fdb9e250e1ff428 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=squid-6cec793, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-02T05:18:29.927 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:29 smithi192 podman[77820]: 2024-04-02 05:18:29.664093749 +0000 UTC m=+0.376223950 container start f8d485e98507881d99fc71615f91415ecb91a34fdb8d29924fdb9e250e1ff428 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4-deactivate, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=squid-6cec793) 2024-04-02T05:18:29.927 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:29 smithi192 podman[77820]: 2024-04-02 05:18:29.735612148 +0000 UTC m=+0.447742352 container attach f8d485e98507881d99fc71615f91415ecb91a34fdb8d29924fdb9e250e1ff428 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-6cec793, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-04-02T05:18:30.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:30 smithi086 ceph-mon[88839]: failed to encode map e139 with expected crc 2024-04-02T05:18:30.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:30 smithi086 ceph-mon[88839]: failed to encode map e139 with expected crc 2024-04-02T05:18:30.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:30 smithi086 ceph-mon[88839]: failed to encode map e139 with expected crc 2024-04-02T05:18:30.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:30 smithi086 ceph-mon[86443]: failed to encode map e139 with expected crc 2024-04-02T05:18:30.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:30 smithi086 ceph-mon[86443]: failed to encode map e139 with expected crc 2024-04-02T05:18:30.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:30 smithi086 ceph-mon[86443]: failed to encode map e139 with expected crc 2024-04-02T05:18:30.427 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:29 smithi192 podman[77820]: 2024-04-02 05:18:29.964631933 +0000 UTC m=+0.676762136 container died f8d485e98507881d99fc71615f91415ecb91a34fdb8d29924fdb9e250e1ff428 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4-deactivate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, RELEASE=squid-6cec793, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-04-02T05:18:30.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:30 smithi192 ceph-mon[68000]: failed to encode map e139 with expected crc 2024-04-02T05:18:30.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:30 smithi192 ceph-mon[68000]: failed to encode map e139 with expected crc 2024-04-02T05:18:30.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:30 smithi192 ceph-mon[68000]: failed to encode map e139 with expected crc 2024-04-02T05:18:30.722 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:30 smithi192 podman[77847]: 2024-04-02 05:18:30.497898736 +0000 UTC m=+0.527303124 container remove f8d485e98507881d99fc71615f91415ecb91a34fdb8d29924fdb9e250e1ff428 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4-deactivate, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-6cec793) 2024-04-02T05:18:30.722 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:30 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.4.service: Deactivated successfully. 2024-04-02T05:18:30.722 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:30 smithi192 systemd[1]: Stopped Ceph osd.4 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:18:30.722 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:30 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.4.service: Consumed 17.041s CPU time. 2024-04-02T05:18:30.722 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:30 smithi192 systemd[1]: Starting Ceph osd.4 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:18:31.070 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:30 smithi192 podman[77954]: 2024-04-02 05:18:30.83875065 +0000 UTC m=+0.018233943 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:18:31.070 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:30 smithi192 podman[77954]: 2024-04-02 05:18:30.957159439 +0000 UTC m=+0.136642709 container create 08d1253b97c0330487b00096357e664fa6af8cd1e734c7ab3e97a31addf0dbff (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4-activate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=squid-6cec793, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-02T05:18:31.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:31 smithi086 ceph-mon[88839]: pgmap v161: 161 pgs: 7 peering, 19 stale+active+clean, 135 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:18:31.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:31 smithi086 ceph-mon[86443]: pgmap v161: 161 pgs: 7 peering, 19 stale+active+clean, 135 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:18:31.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:31 smithi192 ceph-mon[68000]: pgmap v161: 161 pgs: 7 peering, 19 stale+active+clean, 135 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:18:31.427 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:31 smithi192 podman[77954]: 2024-04-02 05:18:31.206053861 +0000 UTC m=+0.385537132 container init 08d1253b97c0330487b00096357e664fa6af8cd1e734c7ab3e97a31addf0dbff (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4-activate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=squid-6cec793, io.buildah.version=1.33.5, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-04-02T05:18:31.427 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:31 smithi192 podman[77954]: 2024-04-02 05:18:31.209346668 +0000 UTC m=+0.388829942 container start 08d1253b97c0330487b00096357e664fa6af8cd1e734c7ab3e97a31addf0dbff (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4-activate, RELEASE=squid-6cec793, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-04-02T05:18:31.428 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:31 smithi192 podman[77954]: 2024-04-02 05:18:31.272495652 +0000 UTC m=+0.451978942 container attach 08d1253b97c0330487b00096357e664fa6af8cd1e734c7ab3e97a31addf0dbff (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4-activate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=squid-6cec793, io.buildah.version=1.33.5, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-02T05:18:31.928 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:31 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4-activate[77970]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-02T05:18:31.928 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:31 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4-activate[77970]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-02T05:18:31.928 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:31 smithi192 bash[77954]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-02T05:18:31.928 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:31 smithi192 bash[77954]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-02T05:18:31.928 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:31 smithi192 bash[77954]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-02T05:18:31.928 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:31 smithi192 bash[77954]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-02T05:18:31.928 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:31 smithi192 bash[77954]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-04-02T05:18:31.928 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:31 smithi192 bash[77954]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-02T05:18:31.928 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:31 smithi192 bash[77954]: --> ceph-volume raw activate successful for osd ID: 4 2024-04-02T05:18:31.928 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:31 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4-activate[77970]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-02T05:18:31.929 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:31 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4-activate[77970]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-02T05:18:31.929 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:31 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4-activate[77970]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-04-02T05:18:31.929 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:31 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4-activate[77970]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-02T05:18:31.929 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:31 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4-activate[77970]: --> ceph-volume raw activate successful for osd ID: 4 2024-04-02T05:18:31.929 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:31 smithi192 podman[77954]: 2024-04-02 05:18:31.674508058 +0000 UTC m=+0.853991321 container died 08d1253b97c0330487b00096357e664fa6af8cd1e734c7ab3e97a31addf0dbff (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4-activate, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=squid-6cec793, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-04-02T05:18:32.311 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:32 smithi192 ceph-mon[68000]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-04-02T05:18:32.332 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:32 smithi086 ceph-mon[86443]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-04-02T05:18:32.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:32 smithi086 ceph-mon[88839]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-04-02T05:18:32.617 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:32 smithi192 podman[78102]: 2024-04-02 05:18:32.309844454 +0000 UTC m=+0.626053570 container remove 08d1253b97c0330487b00096357e664fa6af8cd1e734c7ab3e97a31addf0dbff (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4-activate, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=squid-6cec793) 2024-04-02T05:18:32.617 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:32 smithi192 podman[78168]: 2024-04-02 05:18:32.510991885 +0000 UTC m=+0.103833295 container create f9c58b761b6244509db53e2be5a3b69be2053c6c5b963a20e44773fb123771c0 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, RELEASE=squid-6cec793, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-02T05:18:32.617 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:32 smithi192 podman[78168]: 2024-04-02 05:18:32.425425123 +0000 UTC m=+0.018266490 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:18:32.927 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:32 smithi192 podman[78168]: 2024-04-02 05:18:32.721875444 +0000 UTC m=+0.314716795 container init f9c58b761b6244509db53e2be5a3b69be2053c6c5b963a20e44773fb123771c0 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=squid-6cec793, io.buildah.version=1.33.5) 2024-04-02T05:18:32.927 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:32 smithi192 podman[78168]: 2024-04-02 05:18:32.725129284 +0000 UTC m=+0.317970641 container start f9c58b761b6244509db53e2be5a3b69be2053c6c5b963a20e44773fb123771c0 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=squid-6cec793) 2024-04-02T05:18:32.927 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:32 smithi192 ceph-osd[78188]: -- 172.21.15.192:0/166791527 <== mon.2 v2:172.21.15.192:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5615bae81680 con 0x5615bae4b400 2024-04-02T05:18:32.927 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:32 smithi192 bash[78168]: f9c58b761b6244509db53e2be5a3b69be2053c6c5b963a20e44773fb123771c0 2024-04-02T05:18:32.927 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:32 smithi192 systemd[1]: Started Ceph osd.4 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:18:33.403 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:33 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:18:33.403 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:33 smithi086 ceph-mon[86443]: pgmap v162: 161 pgs: 14 active+undersized+degraded, 7 peering, 8 stale+active+clean, 22 active+undersized, 110 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 66/723 objects degraded (9.129%) 2024-04-02T05:18:33.403 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:33 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:33.403 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:33 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:33.404 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:33 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:18:33.404 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:33 smithi086 ceph-mon[88839]: pgmap v162: 161 pgs: 14 active+undersized+degraded, 7 peering, 8 stale+active+clean, 22 active+undersized, 110 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 66/723 objects degraded (9.129%) 2024-04-02T05:18:33.404 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:33 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:33.404 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:33 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:33.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:33 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:18:33.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:33 smithi192 ceph-mon[68000]: pgmap v162: 161 pgs: 14 active+undersized+degraded, 7 peering, 8 stale+active+clean, 22 active+undersized, 110 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 66/723 objects degraded (9.129%) 2024-04-02T05:18:33.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:33 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:33.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:33 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:33.428 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:33 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4[78184]: 2024-04-02T05:18:33.277+0000 7fa4f2e317c0 -1 Falling back to public interface 2024-04-02T05:18:33.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:18:33 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:18:33] "GET /metrics HTTP/1.1" 200 38194 "" "Prometheus/2.43.0" 2024-04-02T05:18:34.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:34 smithi192 ceph-mon[68000]: Health check failed: Degraded data redundancy: 66/723 objects degraded (9.129%), 14 pgs degraded (PG_DEGRADED) 2024-04-02T05:18:34.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:34 smithi086 ceph-mon[86443]: Health check failed: Degraded data redundancy: 66/723 objects degraded (9.129%), 14 pgs degraded (PG_DEGRADED) 2024-04-02T05:18:34.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:34 smithi086 ceph-mon[88839]: Health check failed: Degraded data redundancy: 66/723 objects degraded (9.129%), 14 pgs degraded (PG_DEGRADED) 2024-04-02T05:18:35.370 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:35 smithi192 ceph-mon[68000]: pgmap v163: 161 pgs: 23 active+undersized+degraded, 40 active+undersized, 98 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 94/723 objects degraded (13.001%) 2024-04-02T05:18:35.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:35 smithi086 ceph-mon[86443]: pgmap v163: 161 pgs: 23 active+undersized+degraded, 40 active+undersized, 98 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 94/723 objects degraded (13.001%) 2024-04-02T05:18:35.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:35 smithi086 ceph-mon[88839]: pgmap v163: 161 pgs: 23 active+undersized+degraded, 40 active+undersized, 98 active+clean; 585 KiB data, 175 MiB used, 715 GiB / 715 GiB avail; 94/723 objects degraded (13.001%) 2024-04-02T05:18:36.259 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:36 smithi192 ceph-mon[68000]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-04-02T05:18:36.259 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:36 smithi192 ceph-mon[68000]: from='osd.4 [v2:172.21.15.192:6800/2751901240,v1:172.21.15.192:6801/2751901240]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-02T05:18:36.260 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:36 smithi192 ceph-mon[68000]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-02T05:18:36.260 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:35 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4[78184]: 2024-04-02T05:18:35.974+0000 7fa4f2e317c0 -1 osd.4 0 read_superblock omap replica is missing. 2024-04-02T05:18:36.260 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:36 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4[78184]: 2024-04-02T05:18:36.002+0000 7fa4f2e317c0 -1 osd.4 137 log_to_monitors true 2024-04-02T05:18:36.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:36 smithi086 ceph-mon[86443]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-04-02T05:18:36.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:36 smithi086 ceph-mon[86443]: from='osd.4 [v2:172.21.15.192:6800/2751901240,v1:172.21.15.192:6801/2751901240]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-02T05:18:36.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:36 smithi086 ceph-mon[86443]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-02T05:18:36.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:36 smithi086 ceph-mon[88839]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-04-02T05:18:36.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:36 smithi086 ceph-mon[88839]: from='osd.4 [v2:172.21.15.192:6800/2751901240,v1:172.21.15.192:6801/2751901240]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-02T05:18:36.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:36 smithi086 ceph-mon[88839]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-02T05:18:36.572 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:18:36 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4[78184]: 2024-04-02T05:18:36.569+0000 7fa4e0281700 -1 osd.4 137 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T05:18:37.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:37 smithi192 ceph-mon[68000]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-02T05:18:37.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:37 smithi192 ceph-mon[68000]: failed to encode map e140 with expected crc 2024-04-02T05:18:37.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:37 smithi192 ceph-mon[68000]: osdmap e140: 8 total, 7 up, 8 in 2024-04-02T05:18:37.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:37 smithi192 ceph-mon[68000]: from='osd.4 [v2:172.21.15.192:6800/2751901240,v1:172.21.15.192:6801/2751901240]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T05:18:37.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:37 smithi192 ceph-mon[68000]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T05:18:37.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:37 smithi192 ceph-mon[68000]: pgmap v165: 161 pgs: 23 active+undersized+degraded, 40 active+undersized, 98 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 94/723 objects degraded (13.001%) 2024-04-02T05:18:37.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:37 smithi192 ceph-mon[68000]: from='osd.4 ' entity='osd.4' 2024-04-02T05:18:37.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:37 smithi086 ceph-mon[86443]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-02T05:18:37.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:37 smithi086 ceph-mon[86443]: failed to encode map e140 with expected crc 2024-04-02T05:18:37.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:37 smithi086 ceph-mon[86443]: osdmap e140: 8 total, 7 up, 8 in 2024-04-02T05:18:37.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:37 smithi086 ceph-mon[86443]: from='osd.4 [v2:172.21.15.192:6800/2751901240,v1:172.21.15.192:6801/2751901240]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T05:18:37.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:37 smithi086 ceph-mon[86443]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T05:18:37.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:37 smithi086 ceph-mon[86443]: pgmap v165: 161 pgs: 23 active+undersized+degraded, 40 active+undersized, 98 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 94/723 objects degraded (13.001%) 2024-04-02T05:18:37.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:37 smithi086 ceph-mon[86443]: from='osd.4 ' entity='osd.4' 2024-04-02T05:18:37.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:37 smithi086 ceph-mon[88839]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-02T05:18:37.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:37 smithi086 ceph-mon[88839]: failed to encode map e140 with expected crc 2024-04-02T05:18:37.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:37 smithi086 ceph-mon[88839]: osdmap e140: 8 total, 7 up, 8 in 2024-04-02T05:18:37.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:37 smithi086 ceph-mon[88839]: from='osd.4 [v2:172.21.15.192:6800/2751901240,v1:172.21.15.192:6801/2751901240]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T05:18:37.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:37 smithi086 ceph-mon[88839]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T05:18:37.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:37 smithi086 ceph-mon[88839]: pgmap v165: 161 pgs: 23 active+undersized+degraded, 40 active+undersized, 98 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 94/723 objects degraded (13.001%) 2024-04-02T05:18:37.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:37 smithi086 ceph-mon[88839]: from='osd.4 ' entity='osd.4' 2024-04-02T05:18:38.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:38 smithi192 ceph-mon[68000]: failed to encode map e140 with expected crc 2024-04-02T05:18:38.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:38 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:38.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:38 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:38.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:38 smithi192 ceph-mon[68000]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-02T05:18:38.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:38 smithi192 ceph-mon[68000]: osd.4 [v2:172.21.15.192:6800/2751901240,v1:172.21.15.192:6801/2751901240] boot 2024-04-02T05:18:38.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:38 smithi192 ceph-mon[68000]: osdmap e141: 8 total, 8 up, 8 in 2024-04-02T05:18:38.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:38 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T05:18:38.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:38 smithi192 ceph-mon[68000]: failed to encode map e141 with expected crc 2024-04-02T05:18:38.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:38 smithi086 ceph-mon[86443]: failed to encode map e140 with expected crc 2024-04-02T05:18:38.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:38 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:38.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:38 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:38.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:38 smithi086 ceph-mon[86443]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-02T05:18:38.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:38 smithi086 ceph-mon[86443]: osd.4 [v2:172.21.15.192:6800/2751901240,v1:172.21.15.192:6801/2751901240] boot 2024-04-02T05:18:38.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:38 smithi086 ceph-mon[86443]: osdmap e141: 8 total, 8 up, 8 in 2024-04-02T05:18:38.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:38 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T05:18:38.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:38 smithi086 ceph-mon[86443]: failed to encode map e141 with expected crc 2024-04-02T05:18:38.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:38 smithi086 ceph-mon[88839]: failed to encode map e140 with expected crc 2024-04-02T05:18:38.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:38 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:38.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:38 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:38.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:38 smithi086 ceph-mon[88839]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-02T05:18:38.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:38 smithi086 ceph-mon[88839]: osd.4 [v2:172.21.15.192:6800/2751901240,v1:172.21.15.192:6801/2751901240] boot 2024-04-02T05:18:38.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:38 smithi086 ceph-mon[88839]: osdmap e141: 8 total, 8 up, 8 in 2024-04-02T05:18:38.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:38 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T05:18:38.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:38 smithi086 ceph-mon[88839]: failed to encode map e141 with expected crc 2024-04-02T05:18:39.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:39 smithi192 ceph-mon[68000]: failed to encode map e141 with expected crc 2024-04-02T05:18:39.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:39 smithi192 ceph-mon[68000]: failed to encode map e141 with expected crc 2024-04-02T05:18:39.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:39 smithi192 ceph-mon[68000]: failed to encode map e141 with expected crc 2024-04-02T05:18:39.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:39 smithi192 ceph-mon[68000]: pgmap v167: 161 pgs: 23 active+undersized+degraded, 40 active+undersized, 98 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 94/723 objects degraded (13.001%) 2024-04-02T05:18:39.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:39 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:39.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:39 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:39.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:39 smithi192 ceph-mon[68000]: failed to encode map e142 with expected crc 2024-04-02T05:18:39.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:39 smithi192 ceph-mon[68000]: osdmap e142: 8 total, 8 up, 8 in 2024-04-02T05:18:39.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:39 smithi086 ceph-mon[88839]: failed to encode map e141 with expected crc 2024-04-02T05:18:39.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:39 smithi086 ceph-mon[88839]: failed to encode map e141 with expected crc 2024-04-02T05:18:39.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:39 smithi086 ceph-mon[88839]: failed to encode map e141 with expected crc 2024-04-02T05:18:39.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:39 smithi086 ceph-mon[88839]: pgmap v167: 161 pgs: 23 active+undersized+degraded, 40 active+undersized, 98 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 94/723 objects degraded (13.001%) 2024-04-02T05:18:39.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:39 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:39.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:39 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:39.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:39 smithi086 ceph-mon[88839]: failed to encode map e142 with expected crc 2024-04-02T05:18:39.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:39 smithi086 ceph-mon[88839]: osdmap e142: 8 total, 8 up, 8 in 2024-04-02T05:18:39.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:39 smithi086 ceph-mon[86443]: failed to encode map e141 with expected crc 2024-04-02T05:18:39.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:39 smithi086 ceph-mon[86443]: failed to encode map e141 with expected crc 2024-04-02T05:18:39.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:39 smithi086 ceph-mon[86443]: failed to encode map e141 with expected crc 2024-04-02T05:18:39.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:39 smithi086 ceph-mon[86443]: pgmap v167: 161 pgs: 23 active+undersized+degraded, 40 active+undersized, 98 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 94/723 objects degraded (13.001%) 2024-04-02T05:18:39.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:39 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:39.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:39 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:39.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:39 smithi086 ceph-mon[86443]: failed to encode map e142 with expected crc 2024-04-02T05:18:39.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:39 smithi086 ceph-mon[86443]: osdmap e142: 8 total, 8 up, 8 in 2024-04-02T05:18:40.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:40 smithi192 ceph-mon[68000]: failed to encode map e142 with expected crc 2024-04-02T05:18:40.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:40 smithi192 ceph-mon[68000]: failed to encode map e142 with expected crc 2024-04-02T05:18:40.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:40 smithi192 ceph-mon[68000]: failed to encode map e142 with expected crc 2024-04-02T05:18:40.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:40 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:40.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:40 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:18:40.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:40 smithi086 ceph-mon[88839]: failed to encode map e142 with expected crc 2024-04-02T05:18:40.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:40 smithi086 ceph-mon[88839]: failed to encode map e142 with expected crc 2024-04-02T05:18:40.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:40 smithi086 ceph-mon[88839]: failed to encode map e142 with expected crc 2024-04-02T05:18:40.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:40 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:40.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:40 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:18:40.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:40 smithi086 ceph-mon[86443]: failed to encode map e142 with expected crc 2024-04-02T05:18:40.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:40 smithi086 ceph-mon[86443]: failed to encode map e142 with expected crc 2024-04-02T05:18:40.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:40 smithi086 ceph-mon[86443]: failed to encode map e142 with expected crc 2024-04-02T05:18:40.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:40 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:40.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:40 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:18:41.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:41 smithi192 ceph-mon[68000]: pgmap v169: 161 pgs: 5 peering, 21 active+undersized+degraded, 37 active+undersized, 98 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 87/723 objects degraded (12.033%) 2024-04-02T05:18:41.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:41 smithi192 ceph-mon[68000]: Health check update: Degraded data redundancy: 87/723 objects degraded (12.033%), 21 pgs degraded (PG_DEGRADED) 2024-04-02T05:18:41.832 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:41 smithi086 ceph-mon[88839]: pgmap v169: 161 pgs: 5 peering, 21 active+undersized+degraded, 37 active+undersized, 98 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 87/723 objects degraded (12.033%) 2024-04-02T05:18:41.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:41 smithi086 ceph-mon[88839]: Health check update: Degraded data redundancy: 87/723 objects degraded (12.033%), 21 pgs degraded (PG_DEGRADED) 2024-04-02T05:18:41.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:41 smithi086 ceph-mon[86443]: pgmap v169: 161 pgs: 5 peering, 21 active+undersized+degraded, 37 active+undersized, 98 active+clean; 585 KiB data, 186 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 87/723 objects degraded (12.033%) 2024-04-02T05:18:41.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:41 smithi086 ceph-mon[86443]: Health check update: Degraded data redundancy: 87/723 objects degraded (12.033%), 21 pgs degraded (PG_DEGRADED) 2024-04-02T05:18:42.333 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:18:41 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:18:41.916+0000 7fe19d012700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-04-02T05:18:43.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:42 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:18:43.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:42 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:43.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:42 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:43.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:42 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:18:43.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:42 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:18:43.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:42 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:43.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:42 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:18:43.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:42 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:18:43.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:42 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:18:43.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:42 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:18:43.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:42 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-02T05:18:43.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:42 smithi086 ceph-mon[86443]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-02T05:18:43.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:42 smithi086 ceph-mon[86443]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-04-02T05:18:43.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:42 smithi086 ceph-mon[86443]: pgmap v170: 161 pgs: 5 peering, 4 active+undersized+degraded, 8 active+undersized, 144 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 823 B/s rd, 0 op/s; 18/723 objects degraded (2.490%) 2024-04-02T05:18:43.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:42 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:18:43.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:42 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:43.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:42 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:43.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:42 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:18:43.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:42 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:18:43.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:42 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:43.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:42 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:18:43.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:42 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:18:43.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:42 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:18:43.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:42 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:18:43.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:42 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-02T05:18:43.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:42 smithi086 ceph-mon[88839]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-02T05:18:43.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:42 smithi086 ceph-mon[88839]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-04-02T05:18:43.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:42 smithi086 ceph-mon[88839]: pgmap v170: 161 pgs: 5 peering, 4 active+undersized+degraded, 8 active+undersized, 144 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 823 B/s rd, 0 op/s; 18/723 objects degraded (2.490%) 2024-04-02T05:18:43.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:42 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:18:43.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:42 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:43.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:42 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:43.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:42 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:18:43.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:42 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:18:43.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:42 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:43.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:42 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:18:43.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:42 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:18:43.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:42 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:18:43.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:42 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:18:43.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:42 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-02T05:18:43.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:42 smithi192 ceph-mon[68000]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-02T05:18:43.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:42 smithi192 ceph-mon[68000]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-04-02T05:18:43.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:42 smithi192 ceph-mon[68000]: pgmap v170: 161 pgs: 5 peering, 4 active+undersized+degraded, 8 active+undersized, 144 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 823 B/s rd, 0 op/s; 18/723 objects degraded (2.490%) 2024-04-02T05:18:43.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:18:43 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:18:43] "GET /metrics HTTP/1.1" 200 38059 "" "Prometheus/2.43.0" 2024-04-02T05:18:44.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:44 smithi192 ceph-mon[68000]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:18:44.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:44 smithi192 ceph-mon[68000]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 18/723 objects degraded (2.490%), 4 pgs degraded) 2024-04-02T05:18:44.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:44 smithi192 ceph-mon[68000]: Cluster is now healthy 2024-04-02T05:18:44.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:44 smithi086 ceph-mon[86443]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:18:44.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:44 smithi086 ceph-mon[86443]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 18/723 objects degraded (2.490%), 4 pgs degraded) 2024-04-02T05:18:44.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:44 smithi086 ceph-mon[86443]: Cluster is now healthy 2024-04-02T05:18:44.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:44 smithi086 ceph-mon[88839]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:18:44.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:44 smithi086 ceph-mon[88839]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 18/723 objects degraded (2.490%), 4 pgs degraded) 2024-04-02T05:18:44.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:44 smithi086 ceph-mon[88839]: Cluster is now healthy 2024-04-02T05:18:47.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:47 smithi192 ceph-mon[68000]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 585 B/s rd, 0 op/s 2024-04-02T05:18:47.832 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:47 smithi086 ceph-mon[86443]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 585 B/s rd, 0 op/s 2024-04-02T05:18:47.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:47 smithi086 ceph-mon[88839]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 585 B/s rd, 0 op/s 2024-04-02T05:18:49.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:49 smithi192 ceph-mon[68000]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:18:49.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:49 smithi086 ceph-mon[88839]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:18:49.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:49 smithi086 ceph-mon[86443]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:18:51.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:51 smithi192 ceph-mon[68000]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 872 B/s rd, 0 op/s 2024-04-02T05:18:51.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:51 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:51.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:51 smithi086 ceph-mon[86443]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 872 B/s rd, 0 op/s 2024-04-02T05:18:51.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:51 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:51.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:51 smithi086 ceph-mon[88839]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 872 B/s rd, 0 op/s 2024-04-02T05:18:51.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:51 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:51.949 INFO:teuthology.orchestra.run.smithi086.stdout:true 2024-04-02T05:18:52.599 INFO:teuthology.orchestra.run.smithi086.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T05:18:52.599 INFO:teuthology.orchestra.run.smithi086.stdout:alertmanager.a smithi086 *:9093,9094 running (6m) 49s ago 28m 26.1M - 0.25.0 c8568f914cd2 660ea62bb6b9 2024-04-02T05:18:52.599 INFO:teuthology.orchestra.run.smithi086.stdout:grafana.a smithi192 *:3000 running (6m) 15s ago 28m 79.5M - 9.4.12 2bacad6d85d8 259af9e63aa5 2024-04-02T05:18:52.599 INFO:teuthology.orchestra.run.smithi086.stdout:iscsi.foo.smithi086.rsbqwx smithi086 running (6m) 49s ago 27m 62.1M - 3.5 e1d6a67b021e fe728eb940f7 2024-04-02T05:18:52.599 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.x smithi192 *:8443,9283,8765 running (8m) 15s ago 33m 453M - 19.0.0-1979-g6cec793d 78f830190839 dd1e6d0f0a95 2024-04-02T05:18:52.599 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.y smithi086 *:8443,9283,8765 running (17m) 49s ago 34m 507M - 19.0.0-1979-g6cec793d 78f830190839 3dca54a018ab 2024-04-02T05:18:52.600 INFO:teuthology.orchestra.run.smithi086.stdout:mon.a smithi086 running (4m) 49s ago 35m 51.7M 2048M 19.0.0-1979-g6cec793d 78f830190839 8e478dd4da2d 2024-04-02T05:18:52.600 INFO:teuthology.orchestra.run.smithi086.stdout:mon.b smithi192 running (5m) 15s ago 33m 46.5M 2048M 19.0.0-1979-g6cec793d 78f830190839 4125e997188f 2024-04-02T05:18:52.600 INFO:teuthology.orchestra.run.smithi086.stdout:mon.c smithi086 running (4m) 49s ago 33m 42.9M 2048M 19.0.0-1979-g6cec793d 78f830190839 8180999b3142 2024-04-02T05:18:52.600 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.a smithi086 *:9100 running (7m) 49s ago 29m 16.3M - 1.5.0 0da6a335fe13 05a844569d7b 2024-04-02T05:18:52.600 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.b smithi192 *:9100 running (7m) 15s ago 29m 16.9M - 1.5.0 0da6a335fe13 1a7d6788d58b 2024-04-02T05:18:52.600 INFO:teuthology.orchestra.run.smithi086.stdout:osd.0 smithi086 running (3m) 49s ago 32m 46.0M 2992M 19.0.0-1979-g6cec793d 78f830190839 e521a0ef7daf 2024-04-02T05:18:52.600 INFO:teuthology.orchestra.run.smithi086.stdout:osd.1 smithi086 running (2m) 49s ago 32m 53.1M 2992M 19.0.0-1979-g6cec793d 78f830190839 76fe72844771 2024-04-02T05:18:52.600 INFO:teuthology.orchestra.run.smithi086.stdout:osd.2 smithi086 running (108s) 49s ago 32m 48.9M 2992M 19.0.0-1979-g6cec793d 78f830190839 28594f089a8f 2024-04-02T05:18:52.600 INFO:teuthology.orchestra.run.smithi086.stdout:osd.3 smithi086 running (54s) 49s ago 31m 12.3M 2992M 19.0.0-1979-g6cec793d 78f830190839 6028f927dfd6 2024-04-02T05:18:52.600 INFO:teuthology.orchestra.run.smithi086.stdout:osd.4 smithi192 running (20s) 15s ago 31m 12.2M 3248M 19.0.0-1979-g6cec793d 78f830190839 f9c58b761b62 2024-04-02T05:18:52.600 INFO:teuthology.orchestra.run.smithi086.stdout:osd.5 smithi192 running (31m) 15s ago 31m 61.4M 3248M 17.2.0 e1d6a67b021e 7259249605f2 2024-04-02T05:18:52.600 INFO:teuthology.orchestra.run.smithi086.stdout:osd.6 smithi192 running (30m) 15s ago 30m 58.3M 3248M 17.2.0 e1d6a67b021e bb7d171b02a1 2024-04-02T05:18:52.601 INFO:teuthology.orchestra.run.smithi086.stdout:osd.7 smithi192 running (30m) 15s ago 30m 62.2M 3248M 17.2.0 e1d6a67b021e 95f5a0353999 2024-04-02T05:18:52.601 INFO:teuthology.orchestra.run.smithi086.stdout:prometheus.a smithi192 *:9095 running (7m) 15s ago 29m 69.5M - 2.43.0 a07b618ecd1d 8378fcdb60e3 2024-04-02T05:18:52.601 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi086.bxzxlp smithi086 *:8000 running (28m) 49s ago 28m 80.0M - 17.2.0 e1d6a67b021e 6725785fbc53 2024-04-02T05:18:52.601 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi192.pmlbot smithi192 *:8000 running (28m) 15s ago 28m 79.7M - 17.2.0 e1d6a67b021e 5ae3638f8b1c 2024-04-02T05:18:52.990 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:18:52.990 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": { 2024-04-02T05:18:52.990 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 3 2024-04-02T05:18:52.990 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:18:52.990 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": { 2024-04-02T05:18:52.990 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 2 2024-04-02T05:18:52.990 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:18:52.990 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": { 2024-04-02T05:18:52.991 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-04-02T05:18:52.991 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 5 2024-04-02T05:18:52.991 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:18:52.991 INFO:teuthology.orchestra.run.smithi086.stdout: "rgw": { 2024-04-02T05:18:52.991 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-02T05:18:52.991 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:18:52.991 INFO:teuthology.orchestra.run.smithi086.stdout: "overall": { 2024-04-02T05:18:52.991 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-04-02T05:18:52.991 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 10 2024-04-02T05:18:52.991 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-04-02T05:18:52.991 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:18:53.317 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:18:53.317 INFO:teuthology.orchestra.run.smithi086.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", 2024-04-02T05:18:53.317 INFO:teuthology.orchestra.run.smithi086.stdout: "in_progress": true, 2024-04-02T05:18:53.317 INFO:teuthology.orchestra.run.smithi086.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-04-02T05:18:53.317 INFO:teuthology.orchestra.run.smithi086.stdout: "services_complete": [], 2024-04-02T05:18:53.317 INFO:teuthology.orchestra.run.smithi086.stdout: "progress": "5/8 daemons upgraded", 2024-04-02T05:18:53.317 INFO:teuthology.orchestra.run.smithi086.stdout: "message": "Currently upgrading osd daemons", 2024-04-02T05:18:53.317 INFO:teuthology.orchestra.run.smithi086.stdout: "is_paused": false 2024-04-02T05:18:53.317 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:18:53.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:18:53 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:18:53] "GET /metrics HTTP/1.1" 200 38059 "" "Prometheus/2.43.0" 2024-04-02T05:18:53.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:53 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:18:53.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:53 smithi086 ceph-mon[88839]: from='client.54471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:18:53.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:53 smithi086 ceph-mon[88839]: from='client.54477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:18:53.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:53 smithi086 ceph-mon[88839]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:18:53.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:53 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/2276532341' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:18:53.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:53 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:18:53.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:53 smithi086 ceph-mon[86443]: from='client.54471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:18:53.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:53 smithi086 ceph-mon[86443]: from='client.54477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:18:53.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:53 smithi086 ceph-mon[86443]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:18:53.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:53 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/2276532341' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:18:53.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:53 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:18:53.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:53 smithi192 ceph-mon[68000]: from='client.54471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:18:53.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:53 smithi192 ceph-mon[68000]: from='client.54477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:18:53.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:53 smithi192 ceph-mon[68000]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:18:53.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:53 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/2276532341' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:18:54.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:54 smithi192 ceph-mon[68000]: from='client.54483 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:18:54.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:54 smithi192 ceph-mon[68000]: from='client.64438 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:18:54.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:54 smithi086 ceph-mon[88839]: from='client.54483 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:18:54.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:54 smithi086 ceph-mon[88839]: from='client.64438 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:18:54.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:54 smithi086 ceph-mon[86443]: from='client.54483 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:18:54.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:54 smithi086 ceph-mon[86443]: from='client.64438 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:18:55.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:55 smithi192 ceph-mon[68000]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:18:55.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:55 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:55.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:55 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:18:55.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:55 smithi086 ceph-mon[86443]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:18:55.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:55 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:55.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:55 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:18:55.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:55 smithi086 ceph-mon[88839]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:18:55.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:55 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:55.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:55 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:18:56.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:56 smithi192 ceph-mon[68000]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:18:56.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:56 smithi086 ceph-mon[86443]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:18:56.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:56 smithi086 ceph-mon[88839]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:18:57.649 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:57 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-02T05:18:57.649 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:57 smithi192 ceph-mon[68000]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-02T05:18:57.649 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:57 smithi192 ceph-mon[68000]: Upgrade: osd.5 is safe to restart 2024-04-02T05:18:57.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:57 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-02T05:18:57.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:57 smithi086 ceph-mon[88839]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-02T05:18:57.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:57 smithi086 ceph-mon[88839]: Upgrade: osd.5 is safe to restart 2024-04-02T05:18:57.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:57 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-02T05:18:57.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:57 smithi086 ceph-mon[86443]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-02T05:18:57.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:57 smithi086 ceph-mon[86443]: Upgrade: osd.5 is safe to restart 2024-04-02T05:18:58.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:58 smithi192 ceph-mon[68000]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:18:58.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:58 smithi086 ceph-mon[86443]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:18:58.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:58 smithi086 ceph-mon[88839]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:18:59.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:59 smithi192 ceph-mon[68000]: Upgrade: Updating osd.5 2024-04-02T05:18:59.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:59 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:59.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:59 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-02T05:18:59.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:59 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:18:59.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:18:59 smithi192 ceph-mon[68000]: Deploying daemon osd.5 on smithi192 2024-04-02T05:18:59.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:59 smithi086 ceph-mon[88839]: Upgrade: Updating osd.5 2024-04-02T05:18:59.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:59 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:59.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:59 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-02T05:18:59.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:59 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:18:59.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:18:59 smithi086 ceph-mon[88839]: Deploying daemon osd.5 on smithi192 2024-04-02T05:18:59.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:59 smithi086 ceph-mon[86443]: Upgrade: Updating osd.5 2024-04-02T05:18:59.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:59 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:18:59.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:59 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-02T05:18:59.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:59 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:18:59.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:18:59 smithi086 ceph-mon[86443]: Deploying daemon osd.5 on smithi192 2024-04-02T05:19:00.638 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:00 smithi192 ceph-mon[68000]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:19:00.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:00 smithi086 ceph-mon[86443]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:19:00.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:00 smithi086 ceph-mon[88839]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:19:01.848 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:01 smithi192 systemd[1]: Stopping Ceph osd.5 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:19:02.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:01 smithi192 ceph-mon[68000]: osd.5 marked itself down and dead 2024-04-02T05:19:02.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:01 smithi192 ceph-mon[68000]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-02T05:19:02.178 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:01 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5[32021]: 2024-04-02T05:19:01.873+0000 7f4f317ab700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T05:19:02.178 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:01 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5[32021]: 2024-04-02T05:19:01.873+0000 7f4f317ab700 -1 osd.5 142 *** Got signal Terminated *** 2024-04-02T05:19:02.178 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:01 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5[32021]: 2024-04-02T05:19:01.873+0000 7f4f317ab700 -1 osd.5 142 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T05:19:02.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:01 smithi086 ceph-mon[88839]: osd.5 marked itself down and dead 2024-04-02T05:19:02.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:01 smithi086 ceph-mon[88839]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-02T05:19:02.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:01 smithi086 ceph-mon[86443]: osd.5 marked itself down and dead 2024-04-02T05:19:02.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:01 smithi086 ceph-mon[86443]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-02T05:19:02.593 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:02 smithi192 podman[82293]: 2024-04-02 05:19:02.303563125 +0000 UTC m=+0.536887571 container died 7259249605f278ba31a3a1c69f500cb3744620df8c2a21c623707081be8940d2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, name=centos-stream, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, release=754, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True) 2024-04-02T05:19:02.927 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:02 smithi192 podman[82293]: 2024-04-02 05:19:02.689913814 +0000 UTC m=+0.923238241 container cleanup 7259249605f278ba31a3a1c69f500cb3744620df8c2a21c623707081be8940d2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, name=centos-stream, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, ceph=True, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64) 2024-04-02T05:19:02.927 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:02 smithi192 bash[82293]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5 2024-04-02T05:19:03.311 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:02 smithi192 podman[82320]: 2024-04-02 05:19:02.958216727 +0000 UTC m=+0.651884436 container remove 7259249605f278ba31a3a1c69f500cb3744620df8c2a21c623707081be8940d2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, vcs-type=git, vendor=Red Hat, Inc., distribution-scope=public, version=8, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, GIT_BRANCH=HEAD, ceph=True, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8) 2024-04-02T05:19:03.311 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:02 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:19:03.312 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:02 smithi192 ceph-mon[68000]: failed to encode map e143 with expected crc 2024-04-02T05:19:03.312 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:02 smithi192 ceph-mon[68000]: failed to encode map e143 with expected crc 2024-04-02T05:19:03.312 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:02 smithi192 ceph-mon[68000]: failed to encode map e143 with expected crc 2024-04-02T05:19:03.312 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:02 smithi192 ceph-mon[68000]: osdmap e143: 8 total, 7 up, 8 in 2024-04-02T05:19:03.312 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:02 smithi192 ceph-mon[68000]: failed to encode map e143 with expected crc 2024-04-02T05:19:03.312 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:02 smithi192 ceph-mon[68000]: pgmap v181: 161 pgs: 6 peering, 20 stale+active+clean, 135 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:19:03.312 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:02 smithi192 ceph-mon[68000]: osdmap e144: 8 total, 7 up, 8 in 2024-04-02T05:19:03.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:02 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:19:03.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:02 smithi086 ceph-mon[86443]: failed to encode map e143 with expected crc 2024-04-02T05:19:03.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:02 smithi086 ceph-mon[86443]: failed to encode map e143 with expected crc 2024-04-02T05:19:03.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:02 smithi086 ceph-mon[86443]: failed to encode map e143 with expected crc 2024-04-02T05:19:03.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:02 smithi086 ceph-mon[86443]: osdmap e143: 8 total, 7 up, 8 in 2024-04-02T05:19:03.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:02 smithi086 ceph-mon[86443]: failed to encode map e143 with expected crc 2024-04-02T05:19:03.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:02 smithi086 ceph-mon[86443]: pgmap v181: 161 pgs: 6 peering, 20 stale+active+clean, 135 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:19:03.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:02 smithi086 ceph-mon[86443]: osdmap e144: 8 total, 7 up, 8 in 2024-04-02T05:19:03.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:02 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:19:03.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:02 smithi086 ceph-mon[88839]: failed to encode map e143 with expected crc 2024-04-02T05:19:03.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:02 smithi086 ceph-mon[88839]: failed to encode map e143 with expected crc 2024-04-02T05:19:03.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:02 smithi086 ceph-mon[88839]: failed to encode map e143 with expected crc 2024-04-02T05:19:03.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:02 smithi086 ceph-mon[88839]: osdmap e143: 8 total, 7 up, 8 in 2024-04-02T05:19:03.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:02 smithi086 ceph-mon[88839]: failed to encode map e143 with expected crc 2024-04-02T05:19:03.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:02 smithi086 ceph-mon[88839]: pgmap v181: 161 pgs: 6 peering, 20 stale+active+clean, 135 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:19:03.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:02 smithi086 ceph-mon[88839]: osdmap e144: 8 total, 7 up, 8 in 2024-04-02T05:19:03.631 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:03 smithi192 podman[82415]: 2024-04-02 05:19:03.309398934 +0000 UTC m=+0.103179756 container create bf17a1a006e72f62ced2b5de1914a71abf9abda44641eb434fb7ae86fcaa3a0e (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-6cec793, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-02T05:19:03.631 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:03 smithi192 podman[82415]: 2024-04-02 05:19:03.22673064 +0000 UTC m=+0.020511482 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:19:03.631 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:03 smithi192 podman[82415]: 2024-04-02 05:19:03.55449593 +0000 UTC m=+0.348276775 container init bf17a1a006e72f62ced2b5de1914a71abf9abda44641eb434fb7ae86fcaa3a0e (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=squid-6cec793, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-02T05:19:03.631 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:03 smithi192 podman[82415]: 2024-04-02 05:19:03.557991099 +0000 UTC m=+0.351771932 container start bf17a1a006e72f62ced2b5de1914a71abf9abda44641eb434fb7ae86fcaa3a0e (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=squid-6cec793, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326) 2024-04-02T05:19:03.631 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:03 smithi192 podman[82415]: 2024-04-02 05:19:03.628626976 +0000 UTC m=+0.422407804 container attach bf17a1a006e72f62ced2b5de1914a71abf9abda44641eb434fb7ae86fcaa3a0e (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5-deactivate, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=squid-6cec793, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-02T05:19:03.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:19:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:19:03] "GET /metrics HTTP/1.1" 200 38197 "" "Prometheus/2.43.0" 2024-04-02T05:19:03.927 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:03 smithi192 podman[82415]: 2024-04-02 05:19:03.848406624 +0000 UTC m=+0.642187451 container died bf17a1a006e72f62ced2b5de1914a71abf9abda44641eb434fb7ae86fcaa3a0e (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5-deactivate, RELEASE=squid-6cec793, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-04-02T05:19:04.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:03 smithi086 ceph-mon[86443]: failed to encode map e144 with expected crc 2024-04-02T05:19:04.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:03 smithi086 ceph-mon[86443]: failed to encode map e144 with expected crc 2024-04-02T05:19:04.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:03 smithi086 ceph-mon[86443]: failed to encode map e144 with expected crc 2024-04-02T05:19:04.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:03 smithi086 ceph-mon[86443]: failed to encode map e144 with expected crc 2024-04-02T05:19:04.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:03 smithi086 ceph-mon[86443]: failed to encode map e144 with expected crc 2024-04-02T05:19:04.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:03 smithi086 ceph-mon[86443]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-04-02T05:19:04.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:03 smithi086 ceph-mon[86443]: failed to encode map e144 with expected crc 2024-04-02T05:19:04.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:03 smithi086 ceph-mon[88839]: failed to encode map e144 with expected crc 2024-04-02T05:19:04.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:03 smithi086 ceph-mon[88839]: failed to encode map e144 with expected crc 2024-04-02T05:19:04.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:03 smithi086 ceph-mon[88839]: failed to encode map e144 with expected crc 2024-04-02T05:19:04.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:03 smithi086 ceph-mon[88839]: failed to encode map e144 with expected crc 2024-04-02T05:19:04.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:03 smithi086 ceph-mon[88839]: failed to encode map e144 with expected crc 2024-04-02T05:19:04.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:03 smithi086 ceph-mon[88839]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-04-02T05:19:04.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:03 smithi086 ceph-mon[88839]: failed to encode map e144 with expected crc 2024-04-02T05:19:04.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:03 smithi192 ceph-mon[68000]: failed to encode map e144 with expected crc 2024-04-02T05:19:04.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:03 smithi192 ceph-mon[68000]: failed to encode map e144 with expected crc 2024-04-02T05:19:04.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:03 smithi192 ceph-mon[68000]: failed to encode map e144 with expected crc 2024-04-02T05:19:04.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:03 smithi192 ceph-mon[68000]: failed to encode map e144 with expected crc 2024-04-02T05:19:04.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:03 smithi192 ceph-mon[68000]: failed to encode map e144 with expected crc 2024-04-02T05:19:04.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:03 smithi192 ceph-mon[68000]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-04-02T05:19:04.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:03 smithi192 ceph-mon[68000]: failed to encode map e144 with expected crc 2024-04-02T05:19:04.902 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:04 smithi192 podman[82442]: 2024-04-02 05:19:04.482821152 +0000 UTC m=+0.629600866 container remove bf17a1a006e72f62ced2b5de1914a71abf9abda44641eb434fb7ae86fcaa3a0e (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-6cec793, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, GIT_CLEAN=True) 2024-04-02T05:19:04.902 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:04 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.5.service: Deactivated successfully. 2024-04-02T05:19:04.902 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:04 smithi192 systemd[1]: Stopped Ceph osd.5 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:19:04.902 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:04 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.5.service: Consumed 17.273s CPU time. 2024-04-02T05:19:04.903 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:04 smithi192 systemd[1]: Starting Ceph osd.5 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:19:05.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:04 smithi192 ceph-mon[68000]: pgmap v183: 161 pgs: 1 active+undersized+degraded, 30 peering, 9 stale+active+clean, 4 active+undersized, 117 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 1/723 objects degraded (0.138%) 2024-04-02T05:19:05.177 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:04 smithi192 podman[82549]: 2024-04-02 05:19:04.900318271 +0000 UTC m=+0.102596306 container create 51e9bd4a90e80444624a9c02a5632b342db07e14f841f4d99b1c4ae26995545c (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-6cec793, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-04-02T05:19:05.177 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:04 smithi192 podman[82549]: 2024-04-02 05:19:04.815477129 +0000 UTC m=+0.017755119 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:19:05.178 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:05 smithi192 podman[82549]: 2024-04-02 05:19:05.136445567 +0000 UTC m=+0.338723556 container init 51e9bd4a90e80444624a9c02a5632b342db07e14f841f4d99b1c4ae26995545c (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5-activate, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-6cec793, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-02T05:19:05.178 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:05 smithi192 podman[82549]: 2024-04-02 05:19:05.139589704 +0000 UTC m=+0.341867694 container start 51e9bd4a90e80444624a9c02a5632b342db07e14f841f4d99b1c4ae26995545c (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5-activate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-6cec793, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, ceph=True) 2024-04-02T05:19:05.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:04 smithi086 ceph-mon[86443]: pgmap v183: 161 pgs: 1 active+undersized+degraded, 30 peering, 9 stale+active+clean, 4 active+undersized, 117 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 1/723 objects degraded (0.138%) 2024-04-02T05:19:05.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:04 smithi086 ceph-mon[88839]: pgmap v183: 161 pgs: 1 active+undersized+degraded, 30 peering, 9 stale+active+clean, 4 active+undersized, 117 active+clean; 585 KiB data, 194 MiB used, 715 GiB / 715 GiB avail; 1/723 objects degraded (0.138%) 2024-04-02T05:19:05.581 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:05 smithi192 podman[82549]: 2024-04-02 05:19:05.256531732 +0000 UTC m=+0.458809760 container attach 51e9bd4a90e80444624a9c02a5632b342db07e14f841f4d99b1c4ae26995545c (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5-activate, org.label-schema.vendor=CentOS, RELEASE=squid-6cec793, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-04-02T05:19:05.892 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:05 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5-activate[82564]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-02T05:19:05.892 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:05 smithi192 bash[82549]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-02T05:19:05.892 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:05 smithi192 bash[82549]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-02T05:19:05.892 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:05 smithi192 bash[82549]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-02T05:19:05.892 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:05 smithi192 bash[82549]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-02T05:19:05.892 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:05 smithi192 bash[82549]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-04-02T05:19:05.892 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:05 smithi192 bash[82549]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-02T05:19:05.892 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:05 smithi192 bash[82549]: --> ceph-volume raw activate successful for osd ID: 5 2024-04-02T05:19:05.893 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:05 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5-activate[82564]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-02T05:19:05.893 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:05 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5-activate[82564]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-02T05:19:05.893 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:05 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5-activate[82564]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-02T05:19:05.893 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:05 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5-activate[82564]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-04-02T05:19:05.893 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:05 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5-activate[82564]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-02T05:19:05.893 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:05 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5-activate[82564]: --> ceph-volume raw activate successful for osd ID: 5 2024-04-02T05:19:05.893 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:05 smithi192 podman[82549]: 2024-04-02 05:19:05.603071383 +0000 UTC m=+0.805349381 container died 51e9bd4a90e80444624a9c02a5632b342db07e14f841f4d99b1c4ae26995545c (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=squid-6cec793, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-02T05:19:06.167 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:05 smithi192 ceph-mon[68000]: Health check failed: Degraded data redundancy: 1/723 objects degraded (0.138%), 1 pg degraded (PG_DEGRADED) 2024-04-02T05:19:06.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:05 smithi086 ceph-mon[86443]: Health check failed: Degraded data redundancy: 1/723 objects degraded (0.138%), 1 pg degraded (PG_DEGRADED) 2024-04-02T05:19:06.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:05 smithi086 ceph-mon[88839]: Health check failed: Degraded data redundancy: 1/723 objects degraded (0.138%), 1 pg degraded (PG_DEGRADED) 2024-04-02T05:19:06.427 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:06 smithi192 podman[82690]: 2024-04-02 05:19:06.165332037 +0000 UTC m=+0.557362231 container remove 51e9bd4a90e80444624a9c02a5632b342db07e14f841f4d99b1c4ae26995545c (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240326, RELEASE=squid-6cec793, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-04-02T05:19:06.427 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:06 smithi192 podman[82757]: 2024-04-02 05:19:06.281130747 +0000 UTC m=+0.019012897 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:19:06.763 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:06 smithi192 podman[82757]: 2024-04-02 05:19:06.4415839 +0000 UTC m=+0.179466065 container create 813afd5cf75d58344b1b028e854689b6efdddb8eea18970acf09c34dc325d1e2 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, ceph=True, RELEASE=squid-6cec793, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-02T05:19:06.763 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:06 smithi192 podman[82757]: 2024-04-02 05:19:06.66107023 +0000 UTC m=+0.398952355 container init 813afd5cf75d58344b1b028e854689b6efdddb8eea18970acf09c34dc325d1e2 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-6cec793, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326) 2024-04-02T05:19:06.763 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:06 smithi192 podman[82757]: 2024-04-02 05:19:06.664706454 +0000 UTC m=+0.402588588 container start 813afd5cf75d58344b1b028e854689b6efdddb8eea18970acf09c34dc325d1e2 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5, RELEASE=squid-6cec793, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-04-02T05:19:06.763 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:06 smithi192 ceph-osd[82776]: -- 172.21.15.192:0/1163321071 <== mon.2 v2:172.21.15.192:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5620a213d680 con 0x5620a2106c00 2024-04-02T05:19:06.764 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:06 smithi192 bash[82757]: 813afd5cf75d58344b1b028e854689b6efdddb8eea18970acf09c34dc325d1e2 2024-04-02T05:19:07.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:06 smithi192 ceph-mon[68000]: pgmap v184: 161 pgs: 10 active+undersized+degraded, 30 peering, 19 active+undersized, 102 active+clean; 585 KiB data, 199 MiB used, 715 GiB / 715 GiB avail; 35/723 objects degraded (4.841%) 2024-04-02T05:19:07.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:06 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:07.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:06 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:07.178 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:06 smithi192 systemd[1]: Started Ceph osd.5 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:19:07.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:06 smithi086 ceph-mon[88839]: pgmap v184: 161 pgs: 10 active+undersized+degraded, 30 peering, 19 active+undersized, 102 active+clean; 585 KiB data, 199 MiB used, 715 GiB / 715 GiB avail; 35/723 objects degraded (4.841%) 2024-04-02T05:19:07.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:06 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:07.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:06 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:07.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:06 smithi086 ceph-mon[86443]: pgmap v184: 161 pgs: 10 active+undersized+degraded, 30 peering, 19 active+undersized, 102 active+clean; 585 KiB data, 199 MiB used, 715 GiB / 715 GiB avail; 35/723 objects degraded (4.841%) 2024-04-02T05:19:07.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:06 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:07.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:06 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:07.626 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:07 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5[82772]: 2024-04-02T05:19:07.212+0000 7f75d64527c0 -1 Falling back to public interface 2024-04-02T05:19:08.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:08 smithi086 ceph-mon[88839]: pgmap v185: 161 pgs: 21 active+undersized+degraded, 38 active+undersized, 102 active+clean; 585 KiB data, 199 MiB used, 715 GiB / 715 GiB avail; 84/723 objects degraded (11.618%) 2024-04-02T05:19:08.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:08 smithi086 ceph-mon[88839]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 7 pgs inactive, 10 pgs peering) 2024-04-02T05:19:08.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:08 smithi086 ceph-mon[86443]: pgmap v185: 161 pgs: 21 active+undersized+degraded, 38 active+undersized, 102 active+clean; 585 KiB data, 199 MiB used, 715 GiB / 715 GiB avail; 84/723 objects degraded (11.618%) 2024-04-02T05:19:08.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:08 smithi086 ceph-mon[86443]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 7 pgs inactive, 10 pgs peering) 2024-04-02T05:19:08.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:08 smithi192 ceph-mon[68000]: pgmap v185: 161 pgs: 21 active+undersized+degraded, 38 active+undersized, 102 active+clean; 585 KiB data, 199 MiB used, 715 GiB / 715 GiB avail; 84/723 objects degraded (11.618%) 2024-04-02T05:19:08.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:08 smithi192 ceph-mon[68000]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 7 pgs inactive, 10 pgs peering) 2024-04-02T05:19:10.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:10 smithi192 ceph-mon[68000]: from='osd.5 [v2:172.21.15.192:6808/816019893,v1:172.21.15.192:6809/816019893]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-02T05:19:10.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:10 smithi192 ceph-mon[68000]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-02T05:19:10.177 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:09 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5[82772]: 2024-04-02T05:19:09.918+0000 7f75d64527c0 -1 osd.5 0 read_superblock omap replica is missing. 2024-04-02T05:19:10.177 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:09 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5[82772]: 2024-04-02T05:19:09.944+0000 7f75d64527c0 -1 osd.5 142 log_to_monitors true 2024-04-02T05:19:10.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:10 smithi086 ceph-mon[88839]: from='osd.5 [v2:172.21.15.192:6808/816019893,v1:172.21.15.192:6809/816019893]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-02T05:19:10.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:10 smithi086 ceph-mon[88839]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-02T05:19:10.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:10 smithi086 ceph-mon[86443]: from='osd.5 [v2:172.21.15.192:6808/816019893,v1:172.21.15.192:6809/816019893]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-02T05:19:10.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:10 smithi086 ceph-mon[86443]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-02T05:19:10.927 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:19:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5[82772]: 2024-04-02T05:19:10.467+0000 7f75c38a2700 -1 osd.5 142 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T05:19:11.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:11 smithi086 ceph-mon[86443]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-02T05:19:11.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:11 smithi086 ceph-mon[86443]: from='osd.5 [v2:172.21.15.192:6808/816019893,v1:172.21.15.192:6809/816019893]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T05:19:11.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:11 smithi086 ceph-mon[86443]: osdmap e145: 8 total, 7 up, 8 in 2024-04-02T05:19:11.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:11 smithi086 ceph-mon[86443]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T05:19:11.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:11 smithi086 ceph-mon[86443]: pgmap v187: 161 pgs: 21 active+undersized+degraded, 38 active+undersized, 102 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 84/723 objects degraded (11.618%) 2024-04-02T05:19:11.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:11 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:11.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:11 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:19:11.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:11 smithi086 ceph-mon[86443]: from='osd.5 ' entity='osd.5' 2024-04-02T05:19:11.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:11 smithi086 ceph-mon[86443]: failed to encode map e145 with expected crc 2024-04-02T05:19:11.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:11 smithi086 ceph-mon[88839]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-02T05:19:11.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:11 smithi086 ceph-mon[88839]: from='osd.5 [v2:172.21.15.192:6808/816019893,v1:172.21.15.192:6809/816019893]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T05:19:11.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:11 smithi086 ceph-mon[88839]: osdmap e145: 8 total, 7 up, 8 in 2024-04-02T05:19:11.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:11 smithi086 ceph-mon[88839]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T05:19:11.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:11 smithi086 ceph-mon[88839]: pgmap v187: 161 pgs: 21 active+undersized+degraded, 38 active+undersized, 102 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 84/723 objects degraded (11.618%) 2024-04-02T05:19:11.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:11 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:11.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:11 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:19:11.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:11 smithi086 ceph-mon[88839]: from='osd.5 ' entity='osd.5' 2024-04-02T05:19:11.335 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:11 smithi086 ceph-mon[88839]: failed to encode map e145 with expected crc 2024-04-02T05:19:11.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:11 smithi192 ceph-mon[68000]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-02T05:19:11.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:11 smithi192 ceph-mon[68000]: from='osd.5 [v2:172.21.15.192:6808/816019893,v1:172.21.15.192:6809/816019893]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T05:19:11.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:11 smithi192 ceph-mon[68000]: osdmap e145: 8 total, 7 up, 8 in 2024-04-02T05:19:11.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:11 smithi192 ceph-mon[68000]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T05:19:11.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:11 smithi192 ceph-mon[68000]: pgmap v187: 161 pgs: 21 active+undersized+degraded, 38 active+undersized, 102 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 84/723 objects degraded (11.618%) 2024-04-02T05:19:11.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:11 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:11.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:11 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:19:11.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:11 smithi192 ceph-mon[68000]: from='osd.5 ' entity='osd.5' 2024-04-02T05:19:11.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:11 smithi192 ceph-mon[68000]: failed to encode map e145 with expected crc 2024-04-02T05:19:12.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:12 smithi086 ceph-mon[86443]: failed to encode map e145 with expected crc 2024-04-02T05:19:12.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:12 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:12.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:12 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:12.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:12 smithi086 ceph-mon[86443]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-02T05:19:12.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:12 smithi086 ceph-mon[86443]: failed to encode map e146 with expected crc 2024-04-02T05:19:12.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:12 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T05:19:12.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:12 smithi086 ceph-mon[86443]: osd.5 [v2:172.21.15.192:6808/816019893,v1:172.21.15.192:6809/816019893] boot 2024-04-02T05:19:12.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:12 smithi086 ceph-mon[86443]: osdmap e146: 8 total, 8 up, 8 in 2024-04-02T05:19:12.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:12 smithi086 ceph-mon[86443]: failed to encode map e146 with expected crc 2024-04-02T05:19:12.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:12 smithi086 ceph-mon[86443]: failed to encode map e146 with expected crc 2024-04-02T05:19:12.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:12 smithi086 ceph-mon[86443]: Health check update: Degraded data redundancy: 84/723 objects degraded (11.618%), 21 pgs degraded (PG_DEGRADED) 2024-04-02T05:19:12.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:12 smithi086 ceph-mon[88839]: failed to encode map e145 with expected crc 2024-04-02T05:19:12.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:12 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:12.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:12 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:12.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:12 smithi086 ceph-mon[88839]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-02T05:19:12.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:12 smithi086 ceph-mon[88839]: failed to encode map e146 with expected crc 2024-04-02T05:19:12.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:12 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T05:19:12.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:12 smithi086 ceph-mon[88839]: osd.5 [v2:172.21.15.192:6808/816019893,v1:172.21.15.192:6809/816019893] boot 2024-04-02T05:19:12.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:12 smithi086 ceph-mon[88839]: osdmap e146: 8 total, 8 up, 8 in 2024-04-02T05:19:12.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:12 smithi086 ceph-mon[88839]: failed to encode map e146 with expected crc 2024-04-02T05:19:12.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:12 smithi086 ceph-mon[88839]: failed to encode map e146 with expected crc 2024-04-02T05:19:12.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:12 smithi086 ceph-mon[88839]: Health check update: Degraded data redundancy: 84/723 objects degraded (11.618%), 21 pgs degraded (PG_DEGRADED) 2024-04-02T05:19:12.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:12 smithi192 ceph-mon[68000]: failed to encode map e145 with expected crc 2024-04-02T05:19:12.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:12 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:12.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:12 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:12.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:12 smithi192 ceph-mon[68000]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-02T05:19:12.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:12 smithi192 ceph-mon[68000]: failed to encode map e146 with expected crc 2024-04-02T05:19:12.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:12 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T05:19:12.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:12 smithi192 ceph-mon[68000]: osd.5 [v2:172.21.15.192:6808/816019893,v1:172.21.15.192:6809/816019893] boot 2024-04-02T05:19:12.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:12 smithi192 ceph-mon[68000]: osdmap e146: 8 total, 8 up, 8 in 2024-04-02T05:19:12.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:12 smithi192 ceph-mon[68000]: failed to encode map e146 with expected crc 2024-04-02T05:19:12.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:12 smithi192 ceph-mon[68000]: failed to encode map e146 with expected crc 2024-04-02T05:19:12.428 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:12 smithi192 ceph-mon[68000]: Health check update: Degraded data redundancy: 84/723 objects degraded (11.618%), 21 pgs degraded (PG_DEGRADED) 2024-04-02T05:19:13.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:13 smithi086 ceph-mon[88839]: failed to encode map e146 with expected crc 2024-04-02T05:19:13.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:13 smithi086 ceph-mon[88839]: failed to encode map e146 with expected crc 2024-04-02T05:19:13.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:13 smithi086 ceph-mon[88839]: failed to encode map e146 with expected crc 2024-04-02T05:19:13.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:13 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:19:13.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:13 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:13.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:13 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:13.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:13 smithi086 ceph-mon[88839]: pgmap v189: 161 pgs: 5 peering, 18 active+undersized+degraded, 36 active+undersized, 102 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 73/723 objects degraded (10.097%) 2024-04-02T05:19:13.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:13 smithi086 ceph-mon[88839]: failed to encode map e147 with expected crc 2024-04-02T05:19:13.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:13 smithi086 ceph-mon[88839]: osdmap e147: 8 total, 8 up, 8 in 2024-04-02T05:19:13.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:13 smithi086 ceph-mon[88839]: failed to encode map e147 with expected crc 2024-04-02T05:19:13.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:13 smithi086 ceph-mon[88839]: failed to encode map e147 with expected crc 2024-04-02T05:19:13.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:13 smithi086 ceph-mon[88839]: failed to encode map e147 with expected crc 2024-04-02T05:19:13.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:13 smithi086 ceph-mon[86443]: failed to encode map e146 with expected crc 2024-04-02T05:19:13.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:13 smithi086 ceph-mon[86443]: failed to encode map e146 with expected crc 2024-04-02T05:19:13.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:13 smithi086 ceph-mon[86443]: failed to encode map e146 with expected crc 2024-04-02T05:19:13.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:13 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:19:13.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:13 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:13.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:13 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:13.334 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:13 smithi086 ceph-mon[86443]: pgmap v189: 161 pgs: 5 peering, 18 active+undersized+degraded, 36 active+undersized, 102 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 73/723 objects degraded (10.097%) 2024-04-02T05:19:13.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:13 smithi086 ceph-mon[86443]: failed to encode map e147 with expected crc 2024-04-02T05:19:13.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:13 smithi086 ceph-mon[86443]: osdmap e147: 8 total, 8 up, 8 in 2024-04-02T05:19:13.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:13 smithi086 ceph-mon[86443]: failed to encode map e147 with expected crc 2024-04-02T05:19:13.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:13 smithi086 ceph-mon[86443]: failed to encode map e147 with expected crc 2024-04-02T05:19:13.335 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:13 smithi086 ceph-mon[86443]: failed to encode map e147 with expected crc 2024-04-02T05:19:13.337 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:13 smithi192 ceph-mon[68000]: failed to encode map e146 with expected crc 2024-04-02T05:19:13.337 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:13 smithi192 ceph-mon[68000]: failed to encode map e146 with expected crc 2024-04-02T05:19:13.338 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:13 smithi192 ceph-mon[68000]: failed to encode map e146 with expected crc 2024-04-02T05:19:13.338 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:13 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:19:13.338 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:13 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:13.338 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:13 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:13.338 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:13 smithi192 ceph-mon[68000]: pgmap v189: 161 pgs: 5 peering, 18 active+undersized+degraded, 36 active+undersized, 102 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s; 73/723 objects degraded (10.097%) 2024-04-02T05:19:13.338 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:13 smithi192 ceph-mon[68000]: failed to encode map e147 with expected crc 2024-04-02T05:19:13.338 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:13 smithi192 ceph-mon[68000]: osdmap e147: 8 total, 8 up, 8 in 2024-04-02T05:19:13.338 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:13 smithi192 ceph-mon[68000]: failed to encode map e147 with expected crc 2024-04-02T05:19:13.338 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:13 smithi192 ceph-mon[68000]: failed to encode map e147 with expected crc 2024-04-02T05:19:13.338 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:13 smithi192 ceph-mon[68000]: failed to encode map e147 with expected crc 2024-04-02T05:19:13.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:19:13 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:19:13] "GET /metrics HTTP/1.1" 200 38053 "" "Prometheus/2.43.0" 2024-04-02T05:19:14.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:14 smithi192 ceph-mon[68000]: failed to encode map e147 with expected crc 2024-04-02T05:19:14.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:14 smithi192 ceph-mon[68000]: failed to encode map e147 with expected crc 2024-04-02T05:19:14.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:14 smithi086 ceph-mon[88839]: failed to encode map e147 with expected crc 2024-04-02T05:19:14.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:14 smithi086 ceph-mon[88839]: failed to encode map e147 with expected crc 2024-04-02T05:19:14.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:14 smithi086 ceph-mon[86443]: failed to encode map e147 with expected crc 2024-04-02T05:19:14.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:14 smithi086 ceph-mon[86443]: failed to encode map e147 with expected crc 2024-04-02T05:19:15.304 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:15 smithi192 ceph-mon[68000]: pgmap v191: 161 pgs: 19 peering, 14 active+undersized+degraded, 22 active+undersized, 106 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 54/723 objects degraded (7.469%) 2024-04-02T05:19:15.530 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:15 smithi086 ceph-mon[86443]: pgmap v191: 161 pgs: 19 peering, 14 active+undersized+degraded, 22 active+undersized, 106 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 54/723 objects degraded (7.469%) 2024-04-02T05:19:15.530 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:15 smithi086 ceph-mon[88839]: pgmap v191: 161 pgs: 19 peering, 14 active+undersized+degraded, 22 active+undersized, 106 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 54/723 objects degraded (7.469%) 2024-04-02T05:19:15.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:19:15 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:19:15.527+0000 7fe19d012700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (19 PGs are or would become offline) 2024-04-02T05:19:16.629 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:16 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:16.629 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:16 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:16.629 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:16 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:19:16.629 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:16 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:19:16.629 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:16 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:16.629 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:16 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:19:16.629 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:16 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:19:16.629 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:16 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:19:16.629 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:16 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:19:16.629 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:16 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-02T05:19:16.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:16 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:16.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:16 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:16.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:16 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:19:16.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:16 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:19:16.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:16 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:16.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:16 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:19:16.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:16 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:19:16.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:16 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:19:16.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:16 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:19:16.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:16 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-02T05:19:16.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:16 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:16.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:16 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:16.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:16 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:19:16.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:16 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:19:16.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:16 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:16.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:16 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:19:16.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:16 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:19:16.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:16 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:19:16.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:16 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:19:16.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:16 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-02T05:19:17.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:17 smithi192 ceph-mon[68000]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-02T05:19:17.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:17 smithi192 ceph-mon[68000]: Upgrade: unsafe to stop osd(s) at this time (19 PGs are or would become offline) 2024-04-02T05:19:17.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:17 smithi192 ceph-mon[68000]: pgmap v192: 161 pgs: 19 peering, 142 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 486 B/s rd, 0 op/s 2024-04-02T05:19:17.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:17 smithi192 ceph-mon[68000]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 54/723 objects degraded (7.469%), 14 pgs degraded) 2024-04-02T05:19:17.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:17 smithi192 ceph-mon[68000]: Cluster is now healthy 2024-04-02T05:19:17.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:17 smithi086 ceph-mon[88839]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-02T05:19:17.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:17 smithi086 ceph-mon[88839]: Upgrade: unsafe to stop osd(s) at this time (19 PGs are or would become offline) 2024-04-02T05:19:17.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:17 smithi086 ceph-mon[88839]: pgmap v192: 161 pgs: 19 peering, 142 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 486 B/s rd, 0 op/s 2024-04-02T05:19:17.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:17 smithi086 ceph-mon[88839]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 54/723 objects degraded (7.469%), 14 pgs degraded) 2024-04-02T05:19:17.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:17 smithi086 ceph-mon[88839]: Cluster is now healthy 2024-04-02T05:19:17.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:17 smithi086 ceph-mon[86443]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-02T05:19:17.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:17 smithi086 ceph-mon[86443]: Upgrade: unsafe to stop osd(s) at this time (19 PGs are or would become offline) 2024-04-02T05:19:17.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:17 smithi086 ceph-mon[86443]: pgmap v192: 161 pgs: 19 peering, 142 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 486 B/s rd, 0 op/s 2024-04-02T05:19:17.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:17 smithi086 ceph-mon[86443]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 54/723 objects degraded (7.469%), 14 pgs degraded) 2024-04-02T05:19:17.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:17 smithi086 ceph-mon[86443]: Cluster is now healthy 2024-04-02T05:19:19.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:19 smithi192 ceph-mon[68000]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:19:19.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:19 smithi086 ceph-mon[86443]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:19:19.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:19 smithi086 ceph-mon[88839]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:19:21.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:21 smithi192 ceph-mon[68000]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 925 B/s rd, 0 op/s 2024-04-02T05:19:21.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:21 smithi086 ceph-mon[86443]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 925 B/s rd, 0 op/s 2024-04-02T05:19:21.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:21 smithi086 ceph-mon[88839]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 925 B/s rd, 0 op/s 2024-04-02T05:19:23.635 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:19:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:19:23] "GET /metrics HTTP/1.1" 200 38053 "" "Prometheus/2.43.0" 2024-04-02T05:19:23.635 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:23 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:19:23.636 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:23 smithi086 ceph-mon[88839]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 818 B/s rd, 0 op/s 2024-04-02T05:19:23.636 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:23 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:19:23.636 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:23 smithi086 ceph-mon[86443]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 818 B/s rd, 0 op/s 2024-04-02T05:19:23.676 INFO:teuthology.orchestra.run.smithi086.stdout:true 2024-04-02T05:19:23.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:23 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:19:23.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:23 smithi192 ceph-mon[68000]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 818 B/s rd, 0 op/s 2024-04-02T05:19:24.320 INFO:teuthology.orchestra.run.smithi086.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T05:19:24.321 INFO:teuthology.orchestra.run.smithi086.stdout:alertmanager.a smithi086 *:9093,9094 running (7m) 81s ago 29m 26.1M - 0.25.0 c8568f914cd2 660ea62bb6b9 2024-04-02T05:19:24.321 INFO:teuthology.orchestra.run.smithi086.stdout:grafana.a smithi192 *:3000 running (7m) 13s ago 29m 80.2M - 9.4.12 2bacad6d85d8 259af9e63aa5 2024-04-02T05:19:24.321 INFO:teuthology.orchestra.run.smithi086.stdout:iscsi.foo.smithi086.rsbqwx smithi086 running (6m) 81s ago 28m 62.1M - 3.5 e1d6a67b021e fe728eb940f7 2024-04-02T05:19:24.321 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.x smithi192 *:8443,9283,8765 running (8m) 13s ago 33m 453M - 19.0.0-1979-g6cec793d 78f830190839 dd1e6d0f0a95 2024-04-02T05:19:24.321 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.y smithi086 *:8443,9283,8765 running (18m) 81s ago 35m 507M - 19.0.0-1979-g6cec793d 78f830190839 3dca54a018ab 2024-04-02T05:19:24.321 INFO:teuthology.orchestra.run.smithi086.stdout:mon.a smithi086 running (5m) 81s ago 35m 51.7M 2048M 19.0.0-1979-g6cec793d 78f830190839 8e478dd4da2d 2024-04-02T05:19:24.321 INFO:teuthology.orchestra.run.smithi086.stdout:mon.b smithi192 running (6m) 13s ago 33m 48.2M 2048M 19.0.0-1979-g6cec793d 78f830190839 4125e997188f 2024-04-02T05:19:24.321 INFO:teuthology.orchestra.run.smithi086.stdout:mon.c smithi086 running (4m) 81s ago 34m 42.9M 2048M 19.0.0-1979-g6cec793d 78f830190839 8180999b3142 2024-04-02T05:19:24.321 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.a smithi086 *:9100 running (8m) 81s ago 30m 16.3M - 1.5.0 0da6a335fe13 05a844569d7b 2024-04-02T05:19:24.321 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.b smithi192 *:9100 running (8m) 13s ago 30m 16.8M - 1.5.0 0da6a335fe13 1a7d6788d58b 2024-04-02T05:19:24.321 INFO:teuthology.orchestra.run.smithi086.stdout:osd.0 smithi086 running (3m) 81s ago 33m 46.0M 2992M 19.0.0-1979-g6cec793d 78f830190839 e521a0ef7daf 2024-04-02T05:19:24.322 INFO:teuthology.orchestra.run.smithi086.stdout:osd.1 smithi086 running (3m) 81s ago 33m 53.1M 2992M 19.0.0-1979-g6cec793d 78f830190839 76fe72844771 2024-04-02T05:19:24.322 INFO:teuthology.orchestra.run.smithi086.stdout:osd.2 smithi086 running (2m) 81s ago 32m 48.9M 2992M 19.0.0-1979-g6cec793d 78f830190839 28594f089a8f 2024-04-02T05:19:24.322 INFO:teuthology.orchestra.run.smithi086.stdout:osd.3 smithi086 running (86s) 81s ago 32m 12.3M 2992M 19.0.0-1979-g6cec793d 78f830190839 6028f927dfd6 2024-04-02T05:19:24.322 INFO:teuthology.orchestra.run.smithi086.stdout:osd.4 smithi192 running (51s) 13s ago 31m 49.8M 3248M 19.0.0-1979-g6cec793d 78f830190839 f9c58b761b62 2024-04-02T05:19:24.322 INFO:teuthology.orchestra.run.smithi086.stdout:osd.5 smithi192 running (18s) 13s ago 31m 12.6M 3248M 19.0.0-1979-g6cec793d 78f830190839 813afd5cf75d 2024-04-02T05:19:24.322 INFO:teuthology.orchestra.run.smithi086.stdout:osd.6 smithi192 running (31m) 13s ago 31m 59.5M 3248M 17.2.0 e1d6a67b021e bb7d171b02a1 2024-04-02T05:19:24.322 INFO:teuthology.orchestra.run.smithi086.stdout:osd.7 smithi192 running (30m) 13s ago 30m 62.9M 3248M 17.2.0 e1d6a67b021e 95f5a0353999 2024-04-02T05:19:24.322 INFO:teuthology.orchestra.run.smithi086.stdout:prometheus.a smithi192 *:9095 running (7m) 13s ago 30m 72.3M - 2.43.0 a07b618ecd1d 8378fcdb60e3 2024-04-02T05:19:24.322 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi086.bxzxlp smithi086 *:8000 running (28m) 81s ago 28m 80.0M - 17.2.0 e1d6a67b021e 6725785fbc53 2024-04-02T05:19:24.322 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi192.pmlbot smithi192 *:8000 running (28m) 13s ago 28m 79.9M - 17.2.0 e1d6a67b021e 5ae3638f8b1c 2024-04-02T05:19:24.710 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:19:24.710 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": { 2024-04-02T05:19:24.711 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 3 2024-04-02T05:19:24.711 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:19:24.711 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": { 2024-04-02T05:19:24.711 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 2 2024-04-02T05:19:24.711 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:19:24.711 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": { 2024-04-02T05:19:24.711 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-04-02T05:19:24.711 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 6 2024-04-02T05:19:24.711 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:19:24.711 INFO:teuthology.orchestra.run.smithi086.stdout: "rgw": { 2024-04-02T05:19:24.711 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-02T05:19:24.711 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:19:24.711 INFO:teuthology.orchestra.run.smithi086.stdout: "overall": { 2024-04-02T05:19:24.711 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-04-02T05:19:24.712 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 11 2024-04-02T05:19:24.712 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-04-02T05:19:24.712 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:19:25.036 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:19:25.036 INFO:teuthology.orchestra.run.smithi086.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", 2024-04-02T05:19:25.036 INFO:teuthology.orchestra.run.smithi086.stdout: "in_progress": true, 2024-04-02T05:19:25.036 INFO:teuthology.orchestra.run.smithi086.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-04-02T05:19:25.036 INFO:teuthology.orchestra.run.smithi086.stdout: "services_complete": [], 2024-04-02T05:19:25.037 INFO:teuthology.orchestra.run.smithi086.stdout: "progress": "6/8 daemons upgraded", 2024-04-02T05:19:25.037 INFO:teuthology.orchestra.run.smithi086.stdout: "message": "Currently upgrading osd daemons", 2024-04-02T05:19:25.037 INFO:teuthology.orchestra.run.smithi086.stdout: "is_paused": false 2024-04-02T05:19:25.037 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:19:25.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:25 smithi192 ceph-mon[68000]: from='client.64447 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:19:25.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:25 smithi192 ceph-mon[68000]: from='client.64453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:19:25.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:25 smithi192 ceph-mon[68000]: from='client.64459 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:19:25.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:25 smithi192 ceph-mon[68000]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 863 B/s rd, 0 op/s 2024-04-02T05:19:25.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:25 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/2787614088' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:19:25.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:25 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:25.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:25 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:19:25.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:25 smithi086 ceph-mon[86443]: from='client.64447 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:19:25.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:25 smithi086 ceph-mon[86443]: from='client.64453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:19:25.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:25 smithi086 ceph-mon[86443]: from='client.64459 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:19:25.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:25 smithi086 ceph-mon[86443]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 863 B/s rd, 0 op/s 2024-04-02T05:19:25.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:25 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/2787614088' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:19:25.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:25 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:25.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:25 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:19:25.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:25 smithi086 ceph-mon[88839]: from='client.64447 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:19:25.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:25 smithi086 ceph-mon[88839]: from='client.64453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:19:25.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:25 smithi086 ceph-mon[88839]: from='client.64459 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:19:25.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:25 smithi086 ceph-mon[88839]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 863 B/s rd, 0 op/s 2024-04-02T05:19:25.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:25 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/2787614088' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:19:25.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:25 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:25.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:25 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:19:26.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:26 smithi192 ceph-mon[68000]: from='client.64471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:19:26.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:26 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:26.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:26 smithi086 ceph-mon[88839]: from='client.64471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:19:26.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:26 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:26.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:26 smithi086 ceph-mon[86443]: from='client.64471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:19:26.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:26 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:27.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:27 smithi192 ceph-mon[68000]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:19:27.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:27 smithi086 ceph-mon[88839]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:19:27.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:27 smithi086 ceph-mon[86443]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:19:29.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:29 smithi192 ceph-mon[68000]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:19:29.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:29 smithi086 ceph-mon[88839]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:19:29.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:29 smithi086 ceph-mon[86443]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:19:30.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:30 smithi086 ceph-mon[88839]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:19:30.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:30 smithi086 ceph-mon[86443]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:19:30.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:30 smithi192 ceph-mon[68000]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:19:31.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:31 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-02T05:19:31.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:31 smithi086 ceph-mon[88839]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-02T05:19:31.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:31 smithi086 ceph-mon[88839]: Upgrade: osd.6 is safe to restart 2024-04-02T05:19:31.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:31 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-02T05:19:31.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:31 smithi086 ceph-mon[86443]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-02T05:19:31.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:31 smithi086 ceph-mon[86443]: Upgrade: osd.6 is safe to restart 2024-04-02T05:19:31.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:31 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-02T05:19:31.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:31 smithi192 ceph-mon[68000]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-02T05:19:31.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:31 smithi192 ceph-mon[68000]: Upgrade: osd.6 is safe to restart 2024-04-02T05:19:33.343 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:33 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:19:33.343 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:33 smithi192 ceph-mon[68000]: Upgrade: Updating osd.6 2024-04-02T05:19:33.343 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:33 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:33.343 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:33 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-02T05:19:33.343 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:33 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:19:33.343 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:33 smithi192 ceph-mon[68000]: Deploying daemon osd.6 on smithi192 2024-04-02T05:19:33.343 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:33 smithi192 ceph-mon[68000]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:19:33.403 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:33 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:19:33.403 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:33 smithi086 ceph-mon[88839]: Upgrade: Updating osd.6 2024-04-02T05:19:33.403 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:33 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:33.403 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:33 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-02T05:19:33.403 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:33 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:19:33.403 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:33 smithi086 ceph-mon[88839]: Deploying daemon osd.6 on smithi192 2024-04-02T05:19:33.403 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:33 smithi086 ceph-mon[88839]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:19:33.404 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:33 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:19:33.404 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:33 smithi086 ceph-mon[86443]: Upgrade: Updating osd.6 2024-04-02T05:19:33.404 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:33 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:33.404 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:33 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-02T05:19:33.404 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:33 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:19:33.404 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:33 smithi086 ceph-mon[86443]: Deploying daemon osd.6 on smithi192 2024-04-02T05:19:33.404 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:33 smithi086 ceph-mon[86443]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:19:33.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:19:33 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:19:33] "GET /metrics HTTP/1.1" 200 38207 "" "Prometheus/2.43.0" 2024-04-02T05:19:35.590 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:35 smithi192 ceph-mon[68000]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:19:35.590 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:35 smithi192 systemd[1]: Stopping Ceph osd.6 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:19:35.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:35 smithi086 ceph-mon[86443]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:19:35.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:35 smithi086 ceph-mon[88839]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:19:35.927 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:35 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[34906]: 2024-04-02T05:19:35.586+0000 7f2cd48ef700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T05:19:35.927 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:35 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[34906]: 2024-04-02T05:19:35.586+0000 7f2cd48ef700 -1 osd.6 147 *** Got signal Terminated *** 2024-04-02T05:19:35.927 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:35 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[34906]: 2024-04-02T05:19:35.586+0000 7f2cd48ef700 -1 osd.6 147 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T05:19:36.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:36 smithi192 ceph-mon[68000]: osd.6 marked itself down and dead 2024-04-02T05:19:36.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:36 smithi086 ceph-mon[86443]: osd.6 marked itself down and dead 2024-04-02T05:19:36.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:36 smithi086 ceph-mon[88839]: osd.6 marked itself down and dead 2024-04-02T05:19:37.032 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:36 smithi192 podman[86887]: 2024-04-02 05:19:36.732666858 +0000 UTC m=+1.340662411 container died bb7d171b02a1d84cc8f5371864417a3bc6a008d5fa89bd6d190d8d085ae2a50d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6, CEPH_POINT_RELEASE=-17.2.0, version=8, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., GIT_CLEAN=True, vcs-type=git, RELEASE=HEAD, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, release=754, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public) 2024-04-02T05:19:37.393 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:37 smithi192 podman[86887]: 2024-04-02 05:19:37.120110024 +0000 UTC m=+1.728105619 container cleanup bb7d171b02a1d84cc8f5371864417a3bc6a008d5fa89bd6d190d8d085ae2a50d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, vendor=Red Hat, Inc., io.openshift.expose-services=, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, name=centos-stream, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, version=8, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git) 2024-04-02T05:19:37.393 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:37 smithi192 bash[86887]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6 2024-04-02T05:19:37.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:37 smithi192 ceph-mon[68000]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:19:37.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:37 smithi192 ceph-mon[68000]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-02T05:19:37.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:37 smithi192 ceph-mon[68000]: failed to encode map e148 with expected crc 2024-04-02T05:19:37.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:37 smithi192 ceph-mon[68000]: osdmap e148: 8 total, 7 up, 8 in 2024-04-02T05:19:37.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:37 smithi192 ceph-mon[68000]: failed to encode map e148 with expected crc 2024-04-02T05:19:37.678 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:37 smithi192 podman[86905]: 2024-04-02 05:19:37.396192666 +0000 UTC m=+0.663914833 container remove bb7d171b02a1d84cc8f5371864417a3bc6a008d5fa89bd6d190d8d085ae2a50d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, GIT_CLEAN=True, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, release=754, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, version=8, io.k8s.display-name=CentOS Stream 8, vcs-type=git, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux ) 2024-04-02T05:19:37.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:37 smithi086 ceph-mon[86443]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:19:37.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:37 smithi086 ceph-mon[86443]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-02T05:19:37.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:37 smithi086 ceph-mon[86443]: failed to encode map e148 with expected crc 2024-04-02T05:19:37.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:37 smithi086 ceph-mon[86443]: osdmap e148: 8 total, 7 up, 8 in 2024-04-02T05:19:37.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:37 smithi086 ceph-mon[86443]: failed to encode map e148 with expected crc 2024-04-02T05:19:37.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:37 smithi086 ceph-mon[88839]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:19:37.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:37 smithi086 ceph-mon[88839]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-02T05:19:37.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:37 smithi086 ceph-mon[88839]: failed to encode map e148 with expected crc 2024-04-02T05:19:37.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:37 smithi086 ceph-mon[88839]: osdmap e148: 8 total, 7 up, 8 in 2024-04-02T05:19:37.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:37 smithi086 ceph-mon[88839]: failed to encode map e148 with expected crc 2024-04-02T05:19:38.060 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:37 smithi192 podman[86994]: 2024-04-02 05:19:37.641130379 +0000 UTC m=+0.018698022 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:19:38.060 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:37 smithi192 podman[86994]: 2024-04-02 05:19:37.77234934 +0000 UTC m=+0.149917017 container create fdcec2a1e205a22684a374a7d18667acf2fdbe0c23034557ca66a4be31a1b313 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6-deactivate, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, RELEASE=squid-6cec793, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-04-02T05:19:38.060 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:37 smithi192 podman[86994]: 2024-04-02 05:19:37.967470593 +0000 UTC m=+0.345038233 container init fdcec2a1e205a22684a374a7d18667acf2fdbe0c23034557ca66a4be31a1b313 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6-deactivate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-6cec793, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=) 2024-04-02T05:19:38.060 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:37 smithi192 podman[86994]: 2024-04-02 05:19:37.970688367 +0000 UTC m=+0.348256004 container start fdcec2a1e205a22684a374a7d18667acf2fdbe0c23034557ca66a4be31a1b313 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6-deactivate, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=squid-6cec793, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-02T05:19:38.060 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:38 smithi192 podman[86994]: 2024-04-02 05:19:38.058401075 +0000 UTC m=+0.435968711 container attach fdcec2a1e205a22684a374a7d18667acf2fdbe0c23034557ca66a4be31a1b313 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6-deactivate, ceph=True, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=squid-6cec793, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-04-02T05:19:38.405 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:38 smithi192 podman[86994]: 2024-04-02 05:19:38.256218931 +0000 UTC m=+0.633786563 container died fdcec2a1e205a22684a374a7d18667acf2fdbe0c23034557ca66a4be31a1b313 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=squid-6cec793) 2024-04-02T05:19:38.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:38 smithi192 ceph-mon[68000]: failed to encode map e149 with expected crc 2024-04-02T05:19:38.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:38 smithi192 ceph-mon[68000]: osdmap e149: 8 total, 7 up, 8 in 2024-04-02T05:19:38.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:38 smithi192 ceph-mon[68000]: failed to encode map e149 with expected crc 2024-04-02T05:19:38.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:38 smithi192 ceph-mon[68000]: failed to encode map e149 with expected crc 2024-04-02T05:19:38.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:38 smithi192 ceph-mon[68000]: failed to encode map e149 with expected crc 2024-04-02T05:19:38.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:38 smithi192 ceph-mon[68000]: pgmap v205: 161 pgs: 2 active+undersized+degraded, 14 peering, 9 stale+active+clean, 4 active+undersized, 132 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 12/723 objects degraded (1.660%) 2024-04-02T05:19:38.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:38 smithi086 ceph-mon[86443]: failed to encode map e149 with expected crc 2024-04-02T05:19:38.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:38 smithi086 ceph-mon[86443]: osdmap e149: 8 total, 7 up, 8 in 2024-04-02T05:19:38.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:38 smithi086 ceph-mon[86443]: failed to encode map e149 with expected crc 2024-04-02T05:19:38.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:38 smithi086 ceph-mon[86443]: failed to encode map e149 with expected crc 2024-04-02T05:19:38.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:38 smithi086 ceph-mon[86443]: failed to encode map e149 with expected crc 2024-04-02T05:19:38.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:38 smithi086 ceph-mon[86443]: pgmap v205: 161 pgs: 2 active+undersized+degraded, 14 peering, 9 stale+active+clean, 4 active+undersized, 132 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 12/723 objects degraded (1.660%) 2024-04-02T05:19:38.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:38 smithi086 ceph-mon[88839]: failed to encode map e149 with expected crc 2024-04-02T05:19:38.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:38 smithi086 ceph-mon[88839]: osdmap e149: 8 total, 7 up, 8 in 2024-04-02T05:19:38.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:38 smithi086 ceph-mon[88839]: failed to encode map e149 with expected crc 2024-04-02T05:19:38.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:38 smithi086 ceph-mon[88839]: failed to encode map e149 with expected crc 2024-04-02T05:19:38.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:38 smithi086 ceph-mon[88839]: failed to encode map e149 with expected crc 2024-04-02T05:19:38.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:38 smithi086 ceph-mon[88839]: pgmap v205: 161 pgs: 2 active+undersized+degraded, 14 peering, 9 stale+active+clean, 4 active+undersized, 132 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 12/723 objects degraded (1.660%) 2024-04-02T05:19:39.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:38 smithi192 podman[87020]: 2024-04-02 05:19:38.870544693 +0000 UTC m=+0.606717966 container remove fdcec2a1e205a22684a374a7d18667acf2fdbe0c23034557ca66a4be31a1b313 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=squid-6cec793, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326) 2024-04-02T05:19:39.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:38 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.6.service: Deactivated successfully. 2024-04-02T05:19:39.178 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:38 smithi192 systemd[1]: Stopped Ceph osd.6 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:19:39.178 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:38 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.6.service: Consumed 14.879s CPU time. 2024-04-02T05:19:39.178 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:39 smithi192 systemd[1]: Starting Ceph osd.6 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:19:39.635 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:39 smithi192 ceph-mon[68000]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-04-02T05:19:39.635 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:39 smithi192 ceph-mon[68000]: Health check failed: Degraded data redundancy: 12/723 objects degraded (1.660%), 2 pgs degraded (PG_DEGRADED) 2024-04-02T05:19:39.635 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:39 smithi192 podman[87124]: 2024-04-02 05:19:39.222979333 +0000 UTC m=+0.018212581 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:19:39.635 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:39 smithi192 podman[87124]: 2024-04-02 05:19:39.338758909 +0000 UTC m=+0.133992155 container create 1a724ea5027542c4cd96b21bdd9404fc15a7a5205bd16ca180a52e943e80ef81 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6-activate, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-6cec793, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2) 2024-04-02T05:19:39.636 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:39 smithi192 podman[87124]: 2024-04-02 05:19:39.549843628 +0000 UTC m=+0.345076875 container init 1a724ea5027542c4cd96b21bdd9404fc15a7a5205bd16ca180a52e943e80ef81 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=squid-6cec793, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-04-02T05:19:39.636 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:39 smithi192 podman[87124]: 2024-04-02 05:19:39.552873311 +0000 UTC m=+0.348106566 container start 1a724ea5027542c4cd96b21bdd9404fc15a7a5205bd16ca180a52e943e80ef81 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6-activate, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-6cec793, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-02T05:19:39.636 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:39 smithi192 podman[87124]: 2024-04-02 05:19:39.633061087 +0000 UTC m=+0.428294336 container attach 1a724ea5027542c4cd96b21bdd9404fc15a7a5205bd16ca180a52e943e80ef81 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6-activate, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, RELEASE=squid-6cec793, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-02T05:19:39.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:39 smithi086 ceph-mon[88839]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-04-02T05:19:39.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:39 smithi086 ceph-mon[88839]: Health check failed: Degraded data redundancy: 12/723 objects degraded (1.660%), 2 pgs degraded (PG_DEGRADED) 2024-04-02T05:19:39.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:39 smithi086 ceph-mon[86443]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-04-02T05:19:39.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:39 smithi086 ceph-mon[86443]: Health check failed: Degraded data redundancy: 12/723 objects degraded (1.660%), 2 pgs degraded (PG_DEGRADED) 2024-04-02T05:19:40.322 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:39 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6-activate[87138]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-02T05:19:40.322 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:39 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6-activate[87138]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-02T05:19:40.322 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:39 smithi192 bash[87124]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-02T05:19:40.322 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:39 smithi192 bash[87124]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-02T05:19:40.322 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:39 smithi192 bash[87124]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-02T05:19:40.322 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:39 smithi192 bash[87124]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-02T05:19:40.322 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:39 smithi192 bash[87124]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-04-02T05:19:40.322 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:39 smithi192 bash[87124]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-02T05:19:40.323 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:39 smithi192 bash[87124]: --> ceph-volume raw activate successful for osd ID: 6 2024-04-02T05:19:40.323 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:39 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6-activate[87138]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-02T05:19:40.323 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:39 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6-activate[87138]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-02T05:19:40.323 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:39 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6-activate[87138]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-04-02T05:19:40.323 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:39 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6-activate[87138]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-02T05:19:40.323 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:39 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6-activate[87138]: --> ceph-volume raw activate successful for osd ID: 6 2024-04-02T05:19:40.323 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:40 smithi192 podman[87124]: 2024-04-02 05:19:40.019123027 +0000 UTC m=+0.814356287 container died 1a724ea5027542c4cd96b21bdd9404fc15a7a5205bd16ca180a52e943e80ef81 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6-activate, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240326, RELEASE=squid-6cec793, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-04-02T05:19:41.088 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:40 smithi192 podman[87271]: 2024-04-02 05:19:40.737336272 +0000 UTC m=+0.713035251 container remove 1a724ea5027542c4cd96b21bdd9404fc15a7a5205bd16ca180a52e943e80ef81 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6-activate, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-6cec793, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-04-02T05:19:41.088 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:40 smithi192 podman[87337]: 2024-04-02 05:19:40.857000431 +0000 UTC m=+0.018107139 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:19:41.088 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:40 smithi192 podman[87337]: 2024-04-02 05:19:40.971536375 +0000 UTC m=+0.132643071 container create d3707b413a43f713fa2c7bb475c6b4f04548786309312c544c1775a22c01d594 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=squid-6cec793, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-04-02T05:19:41.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:41 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:41.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:41 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:19:41.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:41 smithi192 ceph-mon[68000]: pgmap v206: 161 pgs: 6 active+undersized+degraded, 14 peering, 6 stale+active+clean, 10 active+undersized, 125 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 34/723 objects degraded (4.703%) 2024-04-02T05:19:41.428 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:41 smithi192 podman[87337]: 2024-04-02 05:19:41.165924184 +0000 UTC m=+0.327030890 container init d3707b413a43f713fa2c7bb475c6b4f04548786309312c544c1775a22c01d594 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=squid-6cec793, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=) 2024-04-02T05:19:41.428 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:41 smithi192 podman[87337]: 2024-04-02 05:19:41.169609404 +0000 UTC m=+0.330716110 container start d3707b413a43f713fa2c7bb475c6b4f04548786309312c544c1775a22c01d594 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=squid-6cec793, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-04-02T05:19:41.428 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:41 smithi192 ceph-osd[87357]: -- 172.21.15.192:0/1078924775 <== mon.2 v2:172.21.15.192:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56372c599680 con 0x56372c562c00 2024-04-02T05:19:41.428 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:41 smithi192 bash[87337]: d3707b413a43f713fa2c7bb475c6b4f04548786309312c544c1775a22c01d594 2024-04-02T05:19:41.428 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:41 smithi192 systemd[1]: Started Ceph osd.6 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:19:41.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:41 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:41.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:41 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:19:41.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:41 smithi086 ceph-mon[86443]: pgmap v206: 161 pgs: 6 active+undersized+degraded, 14 peering, 6 stale+active+clean, 10 active+undersized, 125 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 34/723 objects degraded (4.703%) 2024-04-02T05:19:41.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:41 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:41.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:41 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:19:41.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:41 smithi086 ceph-mon[88839]: pgmap v206: 161 pgs: 6 active+undersized+degraded, 14 peering, 6 stale+active+clean, 10 active+undersized, 125 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 34/723 objects degraded (4.703%) 2024-04-02T05:19:42.165 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:41 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:19:41.720+0000 7f972be547c0 -1 Falling back to public interface 2024-04-02T05:19:42.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:42 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:42.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:42 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:42.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:42 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:42.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:42 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:42.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:42 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:42.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:42 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:43.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:43 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:19:43.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:43 smithi086 ceph-mon[86443]: pgmap v207: 161 pgs: 14 active+undersized+degraded, 11 peering, 29 active+undersized, 107 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 70/723 objects degraded (9.682%) 2024-04-02T05:19:43.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:19:43 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:19:43] "GET /metrics HTTP/1.1" 200 38282 "" "Prometheus/2.43.0" 2024-04-02T05:19:43.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:43 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:19:43.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:43 smithi086 ceph-mon[88839]: pgmap v207: 161 pgs: 14 active+undersized+degraded, 11 peering, 29 active+undersized, 107 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 70/723 objects degraded (9.682%) 2024-04-02T05:19:43.647 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:43 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:19:43.648 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:43 smithi192 ceph-mon[68000]: pgmap v207: 161 pgs: 14 active+undersized+degraded, 11 peering, 29 active+undersized, 107 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 70/723 objects degraded (9.682%) 2024-04-02T05:19:44.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:44 smithi192 ceph-mon[68000]: pgmap v208: 161 pgs: 18 active+undersized+degraded, 36 active+undersized, 107 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 84/723 objects degraded (11.618%) 2024-04-02T05:19:44.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:44 smithi192 ceph-mon[68000]: Health check update: Degraded data redundancy: 84/723 objects degraded (11.618%), 18 pgs degraded (PG_DEGRADED) 2024-04-02T05:19:44.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:44 smithi192 ceph-mon[68000]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-04-02T05:19:44.678 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:44 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:19:44.446+0000 7f972be547c0 -1 osd.6 0 read_superblock omap replica is missing. 2024-04-02T05:19:44.678 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:44 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:19:44.472+0000 7f972be547c0 -1 osd.6 147 log_to_monitors true 2024-04-02T05:19:44.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:44 smithi086 ceph-mon[86443]: pgmap v208: 161 pgs: 18 active+undersized+degraded, 36 active+undersized, 107 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 84/723 objects degraded (11.618%) 2024-04-02T05:19:44.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:44 smithi086 ceph-mon[86443]: Health check update: Degraded data redundancy: 84/723 objects degraded (11.618%), 18 pgs degraded (PG_DEGRADED) 2024-04-02T05:19:44.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:44 smithi086 ceph-mon[86443]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-04-02T05:19:44.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:44 smithi086 ceph-mon[88839]: pgmap v208: 161 pgs: 18 active+undersized+degraded, 36 active+undersized, 107 active+clean; 585 KiB data, 214 MiB used, 715 GiB / 715 GiB avail; 84/723 objects degraded (11.618%) 2024-04-02T05:19:44.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:44 smithi086 ceph-mon[88839]: Health check update: Degraded data redundancy: 84/723 objects degraded (11.618%), 18 pgs degraded (PG_DEGRADED) 2024-04-02T05:19:44.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:44 smithi086 ceph-mon[88839]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-04-02T05:19:45.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:45 smithi192 ceph-mon[68000]: from='osd.6 [v2:172.21.15.192:6816/1416489286,v1:172.21.15.192:6817/1416489286]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-02T05:19:45.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:45 smithi192 ceph-mon[68000]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-02T05:19:45.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:45 smithi086 ceph-mon[86443]: from='osd.6 [v2:172.21.15.192:6816/1416489286,v1:172.21.15.192:6817/1416489286]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-02T05:19:45.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:45 smithi086 ceph-mon[86443]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-02T05:19:45.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:45 smithi086 ceph-mon[88839]: from='osd.6 [v2:172.21.15.192:6816/1416489286,v1:172.21.15.192:6817/1416489286]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-02T05:19:45.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:45 smithi086 ceph-mon[88839]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-02T05:19:46.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:19:45 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:19:45.884+0000 7f97192a4700 -1 osd.6 147 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T05:19:46.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:46 smithi086 ceph-mon[88839]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-02T05:19:46.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:46 smithi086 ceph-mon[88839]: from='osd.6 [v2:172.21.15.192:6816/1416489286,v1:172.21.15.192:6817/1416489286]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T05:19:46.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:46 smithi086 ceph-mon[88839]: osdmap e150: 8 total, 7 up, 8 in 2024-04-02T05:19:46.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:46 smithi086 ceph-mon[88839]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T05:19:46.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:46 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:46.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:46 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:46.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:46 smithi086 ceph-mon[88839]: from='osd.6 ' entity='osd.6' 2024-04-02T05:19:46.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:46 smithi086 ceph-mon[88839]: pgmap v210: 161 pgs: 18 active+undersized+degraded, 36 active+undersized, 107 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 572 B/s rd, 0 op/s; 84/723 objects degraded (11.618%) 2024-04-02T05:19:46.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:46 smithi086 ceph-mon[86443]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-02T05:19:46.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:46 smithi086 ceph-mon[86443]: from='osd.6 [v2:172.21.15.192:6816/1416489286,v1:172.21.15.192:6817/1416489286]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T05:19:46.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:46 smithi086 ceph-mon[86443]: osdmap e150: 8 total, 7 up, 8 in 2024-04-02T05:19:46.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:46 smithi086 ceph-mon[86443]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T05:19:46.835 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:46 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:46.836 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:46 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:46.836 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:46 smithi086 ceph-mon[86443]: from='osd.6 ' entity='osd.6' 2024-04-02T05:19:46.836 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:46 smithi086 ceph-mon[86443]: pgmap v210: 161 pgs: 18 active+undersized+degraded, 36 active+undersized, 107 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 572 B/s rd, 0 op/s; 84/723 objects degraded (11.618%) 2024-04-02T05:19:46.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:46 smithi192 ceph-mon[68000]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-02T05:19:46.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:46 smithi192 ceph-mon[68000]: from='osd.6 [v2:172.21.15.192:6816/1416489286,v1:172.21.15.192:6817/1416489286]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T05:19:46.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:46 smithi192 ceph-mon[68000]: osdmap e150: 8 total, 7 up, 8 in 2024-04-02T05:19:46.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:46 smithi192 ceph-mon[68000]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T05:19:46.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:46 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:46.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:46 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:46.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:46 smithi192 ceph-mon[68000]: from='osd.6 ' entity='osd.6' 2024-04-02T05:19:46.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:46 smithi192 ceph-mon[68000]: pgmap v210: 161 pgs: 18 active+undersized+degraded, 36 active+undersized, 107 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 572 B/s rd, 0 op/s; 84/723 objects degraded (11.618%) 2024-04-02T05:19:47.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:47 smithi192 ceph-mon[68000]: failed to encode map e150 with expected crc 2024-04-02T05:19:47.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:47 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:47.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:47 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:47.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:47 smithi192 ceph-mon[68000]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-02T05:19:47.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:47 smithi192 ceph-mon[68000]: osd.6 [v2:172.21.15.192:6816/1416489286,v1:172.21.15.192:6817/1416489286] boot 2024-04-02T05:19:47.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:47 smithi192 ceph-mon[68000]: osdmap e151: 8 total, 8 up, 8 in 2024-04-02T05:19:47.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:47 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T05:19:47.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:47 smithi086 ceph-mon[86443]: failed to encode map e150 with expected crc 2024-04-02T05:19:47.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:47 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:47.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:47 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:47.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:47 smithi086 ceph-mon[86443]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-02T05:19:47.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:47 smithi086 ceph-mon[86443]: osd.6 [v2:172.21.15.192:6816/1416489286,v1:172.21.15.192:6817/1416489286] boot 2024-04-02T05:19:47.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:47 smithi086 ceph-mon[86443]: osdmap e151: 8 total, 8 up, 8 in 2024-04-02T05:19:47.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:47 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T05:19:47.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:47 smithi086 ceph-mon[88839]: failed to encode map e150 with expected crc 2024-04-02T05:19:47.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:47 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:47.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:47 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:47.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:47 smithi086 ceph-mon[88839]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-02T05:19:47.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:47 smithi086 ceph-mon[88839]: osd.6 [v2:172.21.15.192:6816/1416489286,v1:172.21.15.192:6817/1416489286] boot 2024-04-02T05:19:47.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:47 smithi086 ceph-mon[88839]: osdmap e151: 8 total, 8 up, 8 in 2024-04-02T05:19:47.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:47 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T05:19:48.671 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:48 smithi192 ceph-mon[68000]: failed to encode map e151 with expected crc 2024-04-02T05:19:48.671 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:48 smithi192 ceph-mon[68000]: failed to encode map e151 with expected crc 2024-04-02T05:19:48.671 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:48 smithi192 ceph-mon[68000]: osdmap e152: 8 total, 8 up, 8 in 2024-04-02T05:19:48.671 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:48 smithi192 ceph-mon[68000]: pgmap v213: 161 pgs: 15 peering, 13 active+undersized+degraded, 26 active+undersized, 107 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s; 59/723 objects degraded (8.160%) 2024-04-02T05:19:48.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:48 smithi086 ceph-mon[86443]: failed to encode map e151 with expected crc 2024-04-02T05:19:48.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:48 smithi086 ceph-mon[86443]: failed to encode map e151 with expected crc 2024-04-02T05:19:48.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:48 smithi086 ceph-mon[86443]: osdmap e152: 8 total, 8 up, 8 in 2024-04-02T05:19:48.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:48 smithi086 ceph-mon[86443]: pgmap v213: 161 pgs: 15 peering, 13 active+undersized+degraded, 26 active+undersized, 107 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s; 59/723 objects degraded (8.160%) 2024-04-02T05:19:48.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:48 smithi086 ceph-mon[88839]: failed to encode map e151 with expected crc 2024-04-02T05:19:48.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:48 smithi086 ceph-mon[88839]: failed to encode map e151 with expected crc 2024-04-02T05:19:48.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:48 smithi086 ceph-mon[88839]: osdmap e152: 8 total, 8 up, 8 in 2024-04-02T05:19:48.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:48 smithi086 ceph-mon[88839]: pgmap v213: 161 pgs: 15 peering, 13 active+undersized+degraded, 26 active+undersized, 107 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s; 59/723 objects degraded (8.160%) 2024-04-02T05:19:49.825 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:49 smithi192 ceph-mon[68000]: failed to encode map e152 with expected crc 2024-04-02T05:19:49.825 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:49 smithi192 ceph-mon[68000]: failed to encode map e152 with expected crc 2024-04-02T05:19:49.825 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:49 smithi192 ceph-mon[68000]: failed to encode map e152 with expected crc 2024-04-02T05:19:49.825 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:49 smithi192 ceph-mon[68000]: failed to encode map e152 with expected crc 2024-04-02T05:19:49.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:49 smithi086 ceph-mon[86443]: failed to encode map e152 with expected crc 2024-04-02T05:19:49.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:49 smithi086 ceph-mon[86443]: failed to encode map e152 with expected crc 2024-04-02T05:19:49.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:49 smithi086 ceph-mon[86443]: failed to encode map e152 with expected crc 2024-04-02T05:19:49.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:49 smithi086 ceph-mon[86443]: failed to encode map e152 with expected crc 2024-04-02T05:19:49.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:49 smithi086 ceph-mon[88839]: failed to encode map e152 with expected crc 2024-04-02T05:19:49.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:49 smithi086 ceph-mon[88839]: failed to encode map e152 with expected crc 2024-04-02T05:19:49.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:49 smithi086 ceph-mon[88839]: failed to encode map e152 with expected crc 2024-04-02T05:19:49.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:49 smithi086 ceph-mon[88839]: failed to encode map e152 with expected crc 2024-04-02T05:19:50.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:19:50 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:19:50.515+0000 7fe19d012700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (19 PGs are or would become offline) 2024-04-02T05:19:51.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:51 smithi192 ceph-mon[68000]: pgmap v214: 161 pgs: 15 peering, 11 active+undersized+degraded, 21 active+undersized, 114 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s; 48/723 objects degraded (6.639%) 2024-04-02T05:19:51.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:51 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:51.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:51 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:51.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:51 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:19:51.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:51 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:19:51.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:51 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:51.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:51 smithi192 ceph-mon[68000]: Health check update: Degraded data redundancy: 48/723 objects degraded (6.639%), 11 pgs degraded (PG_DEGRADED) 2024-04-02T05:19:51.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:51 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:19:51.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:51 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:19:51.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:51 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:19:51.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:51 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:19:51.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:51 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-02T05:19:51.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:51 smithi086 ceph-mon[86443]: pgmap v214: 161 pgs: 15 peering, 11 active+undersized+degraded, 21 active+undersized, 114 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s; 48/723 objects degraded (6.639%) 2024-04-02T05:19:51.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:51 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:51.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:51 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:51.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:51 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:19:51.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:51 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:19:51.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:51 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:51.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:51 smithi086 ceph-mon[86443]: Health check update: Degraded data redundancy: 48/723 objects degraded (6.639%), 11 pgs degraded (PG_DEGRADED) 2024-04-02T05:19:51.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:51 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:19:51.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:51 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:19:51.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:51 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:19:51.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:51 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:19:51.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:51 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-02T05:19:51.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:51 smithi086 ceph-mon[88839]: pgmap v214: 161 pgs: 15 peering, 11 active+undersized+degraded, 21 active+undersized, 114 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s; 48/723 objects degraded (6.639%) 2024-04-02T05:19:51.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:51 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:51.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:51 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:51.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:51 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:19:51.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:51 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:19:51.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:51 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:51.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:51 smithi086 ceph-mon[88839]: Health check update: Degraded data redundancy: 48/723 objects degraded (6.639%), 11 pgs degraded (PG_DEGRADED) 2024-04-02T05:19:51.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:51 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:19:51.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:51 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:19:51.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:51 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:19:51.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:51 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:19:51.835 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:51 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-02T05:19:52.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:52 smithi192 ceph-mon[68000]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-02T05:19:52.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:52 smithi192 ceph-mon[68000]: Upgrade: unsafe to stop osd(s) at this time (19 PGs are or would become offline) 2024-04-02T05:19:52.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:52 smithi086 ceph-mon[86443]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-02T05:19:52.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:52 smithi086 ceph-mon[86443]: Upgrade: unsafe to stop osd(s) at this time (19 PGs are or would become offline) 2024-04-02T05:19:52.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:52 smithi086 ceph-mon[88839]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-02T05:19:52.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:52 smithi086 ceph-mon[88839]: Upgrade: unsafe to stop osd(s) at this time (19 PGs are or would become offline) 2024-04-02T05:19:53.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:53 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:19:53.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:53 smithi192 ceph-mon[68000]: pgmap v215: 161 pgs: 14 peering, 147 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 445 B/s rd, 0 op/s 2024-04-02T05:19:53.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:53 smithi192 ceph-mon[68000]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 48/723 objects degraded (6.639%), 11 pgs degraded) 2024-04-02T05:19:53.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:53 smithi192 ceph-mon[68000]: Cluster is now healthy 2024-04-02T05:19:53.733 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:19:53 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:19:53] "GET /metrics HTTP/1.1" 200 38282 "" "Prometheus/2.43.0" 2024-04-02T05:19:53.734 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:53 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:19:53.734 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:53 smithi086 ceph-mon[88839]: pgmap v215: 161 pgs: 14 peering, 147 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 445 B/s rd, 0 op/s 2024-04-02T05:19:53.734 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:53 smithi086 ceph-mon[88839]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 48/723 objects degraded (6.639%), 11 pgs degraded) 2024-04-02T05:19:53.734 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:53 smithi086 ceph-mon[88839]: Cluster is now healthy 2024-04-02T05:19:53.734 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:53 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:19:53.734 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:53 smithi086 ceph-mon[86443]: pgmap v215: 161 pgs: 14 peering, 147 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 445 B/s rd, 0 op/s 2024-04-02T05:19:53.734 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:53 smithi086 ceph-mon[86443]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 48/723 objects degraded (6.639%), 11 pgs degraded) 2024-04-02T05:19:53.734 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:53 smithi086 ceph-mon[86443]: Cluster is now healthy 2024-04-02T05:19:55.388 INFO:teuthology.orchestra.run.smithi086.stdout:true 2024-04-02T05:19:55.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:55 smithi192 ceph-mon[68000]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:19:55.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:55 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:55.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:55 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:19:55.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:55 smithi086 ceph-mon[86443]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:19:55.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:55 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:55.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:55 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:19:55.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:55 smithi086 ceph-mon[88839]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:19:55.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:55 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:19:55.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:55 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:19:56.034 INFO:teuthology.orchestra.run.smithi086.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T05:19:56.034 INFO:teuthology.orchestra.run.smithi086.stdout:alertmanager.a smithi086 *:9093,9094 running (8m) 113s ago 29m 26.1M - 0.25.0 c8568f914cd2 660ea62bb6b9 2024-04-02T05:19:56.034 INFO:teuthology.orchestra.run.smithi086.stdout:grafana.a smithi192 *:3000 running (7m) 10s ago 29m 77.9M - 9.4.12 2bacad6d85d8 259af9e63aa5 2024-04-02T05:19:56.034 INFO:teuthology.orchestra.run.smithi086.stdout:iscsi.foo.smithi086.rsbqwx smithi086 running (7m) 113s ago 29m 62.1M - 3.5 e1d6a67b021e fe728eb940f7 2024-04-02T05:19:56.034 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.x smithi192 *:8443,9283,8765 running (9m) 10s ago 34m 453M - 19.0.0-1979-g6cec793d 78f830190839 dd1e6d0f0a95 2024-04-02T05:19:56.034 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.y smithi086 *:8443,9283,8765 running (18m) 113s ago 35m 507M - 19.0.0-1979-g6cec793d 78f830190839 3dca54a018ab 2024-04-02T05:19:56.035 INFO:teuthology.orchestra.run.smithi086.stdout:mon.a smithi086 running (5m) 113s ago 36m 51.7M 2048M 19.0.0-1979-g6cec793d 78f830190839 8e478dd4da2d 2024-04-02T05:19:56.035 INFO:teuthology.orchestra.run.smithi086.stdout:mon.b smithi192 running (6m) 10s ago 34m 49.7M 2048M 19.0.0-1979-g6cec793d 78f830190839 4125e997188f 2024-04-02T05:19:56.035 INFO:teuthology.orchestra.run.smithi086.stdout:mon.c smithi086 running (5m) 113s ago 34m 42.9M 2048M 19.0.0-1979-g6cec793d 78f830190839 8180999b3142 2024-04-02T05:19:56.035 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.a smithi086 *:9100 running (8m) 113s ago 30m 16.3M - 1.5.0 0da6a335fe13 05a844569d7b 2024-04-02T05:19:56.035 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.b smithi192 *:9100 running (8m) 10s ago 30m 17.1M - 1.5.0 0da6a335fe13 1a7d6788d58b 2024-04-02T05:19:56.035 INFO:teuthology.orchestra.run.smithi086.stdout:osd.0 smithi086 running (4m) 113s ago 33m 46.0M 2992M 19.0.0-1979-g6cec793d 78f830190839 e521a0ef7daf 2024-04-02T05:19:56.035 INFO:teuthology.orchestra.run.smithi086.stdout:osd.1 smithi086 running (3m) 113s ago 33m 53.1M 2992M 19.0.0-1979-g6cec793d 78f830190839 76fe72844771 2024-04-02T05:19:56.035 INFO:teuthology.orchestra.run.smithi086.stdout:osd.2 smithi086 running (2m) 113s ago 33m 48.9M 2992M 19.0.0-1979-g6cec793d 78f830190839 28594f089a8f 2024-04-02T05:19:56.035 INFO:teuthology.orchestra.run.smithi086.stdout:osd.3 smithi086 running (117s) 113s ago 32m 12.3M 2992M 19.0.0-1979-g6cec793d 78f830190839 6028f927dfd6 2024-04-02T05:19:56.035 INFO:teuthology.orchestra.run.smithi086.stdout:osd.4 smithi192 running (83s) 10s ago 32m 51.5M 3248M 19.0.0-1979-g6cec793d 78f830190839 f9c58b761b62 2024-04-02T05:19:56.035 INFO:teuthology.orchestra.run.smithi086.stdout:osd.5 smithi192 running (49s) 10s ago 32m 51.0M 3248M 19.0.0-1979-g6cec793d 78f830190839 813afd5cf75d 2024-04-02T05:19:56.035 INFO:teuthology.orchestra.run.smithi086.stdout:osd.6 smithi192 running (15s) 10s ago 31m 12.6M 3248M 19.0.0-1979-g6cec793d 78f830190839 d3707b413a43 2024-04-02T05:19:56.035 INFO:teuthology.orchestra.run.smithi086.stdout:osd.7 smithi192 running (31m) 10s ago 31m 64.6M 3248M 17.2.0 e1d6a67b021e 95f5a0353999 2024-04-02T05:19:56.035 INFO:teuthology.orchestra.run.smithi086.stdout:prometheus.a smithi192 *:9095 running (8m) 10s ago 30m 72.4M - 2.43.0 a07b618ecd1d 8378fcdb60e3 2024-04-02T05:19:56.036 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi086.bxzxlp smithi086 *:8000 running (29m) 113s ago 29m 80.0M - 17.2.0 e1d6a67b021e 6725785fbc53 2024-04-02T05:19:56.036 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi192.pmlbot smithi192 *:8000 running (29m) 10s ago 29m 80.1M - 17.2.0 e1d6a67b021e 5ae3638f8b1c 2024-04-02T05:19:56.428 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:19:56.428 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": { 2024-04-02T05:19:56.428 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 3 2024-04-02T05:19:56.428 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:19:56.429 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": { 2024-04-02T05:19:56.429 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 2 2024-04-02T05:19:56.429 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:19:56.429 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": { 2024-04-02T05:19:56.429 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-02T05:19:56.429 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 7 2024-04-02T05:19:56.429 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:19:56.429 INFO:teuthology.orchestra.run.smithi086.stdout: "rgw": { 2024-04-02T05:19:56.429 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-02T05:19:56.429 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:19:56.429 INFO:teuthology.orchestra.run.smithi086.stdout: "overall": { 2024-04-02T05:19:56.429 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-04-02T05:19:56.429 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 12 2024-04-02T05:19:56.429 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-04-02T05:19:56.429 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:19:56.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:56 smithi192 ceph-mon[68000]: from='client.64480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:19:56.754 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:19:56.754 INFO:teuthology.orchestra.run.smithi086.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", 2024-04-02T05:19:56.754 INFO:teuthology.orchestra.run.smithi086.stdout: "in_progress": true, 2024-04-02T05:19:56.754 INFO:teuthology.orchestra.run.smithi086.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-04-02T05:19:56.755 INFO:teuthology.orchestra.run.smithi086.stdout: "services_complete": [], 2024-04-02T05:19:56.755 INFO:teuthology.orchestra.run.smithi086.stdout: "progress": "7/8 daemons upgraded", 2024-04-02T05:19:56.755 INFO:teuthology.orchestra.run.smithi086.stdout: "message": "Currently upgrading osd daemons", 2024-04-02T05:19:56.755 INFO:teuthology.orchestra.run.smithi086.stdout: "is_paused": false 2024-04-02T05:19:56.755 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:19:56.755 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:56 smithi086 ceph-mon[86443]: from='client.64480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:19:56.755 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:56 smithi086 ceph-mon[88839]: from='client.64480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:19:57.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:57 smithi192 ceph-mon[68000]: from='client.54543 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:19:57.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:57 smithi192 ceph-mon[68000]: from='client.64492 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:19:57.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:57 smithi192 ceph-mon[68000]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 866 B/s rd, 0 op/s 2024-04-02T05:19:57.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:57 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/2639191112' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:19:57.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:57 smithi086 ceph-mon[86443]: from='client.54543 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:19:57.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:57 smithi086 ceph-mon[86443]: from='client.64492 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:19:57.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:57 smithi086 ceph-mon[86443]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 866 B/s rd, 0 op/s 2024-04-02T05:19:57.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:57 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/2639191112' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:19:57.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:57 smithi086 ceph-mon[88839]: from='client.54543 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:19:57.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:57 smithi086 ceph-mon[88839]: from='client.64492 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:19:57.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:57 smithi086 ceph-mon[88839]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 866 B/s rd, 0 op/s 2024-04-02T05:19:57.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:57 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/2639191112' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:19:58.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:58 smithi192 ceph-mon[68000]: from='client.64504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:19:58.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:58 smithi086 ceph-mon[88839]: from='client.64504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:19:58.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:58 smithi086 ceph-mon[86443]: from='client.64504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:19:59.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:19:59 smithi192 ceph-mon[68000]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 881 B/s rd, 0 op/s 2024-04-02T05:19:59.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:19:59 smithi086 ceph-mon[88839]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 881 B/s rd, 0 op/s 2024-04-02T05:19:59.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:19:59 smithi086 ceph-mon[86443]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 881 B/s rd, 0 op/s 2024-04-02T05:20:00.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:00 smithi192 ceph-mon[68000]: overall HEALTH_OK 2024-04-02T05:20:00.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:00 smithi086 ceph-mon[88839]: overall HEALTH_OK 2024-04-02T05:20:00.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:00 smithi086 ceph-mon[86443]: overall HEALTH_OK 2024-04-02T05:20:01.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:01 smithi192 ceph-mon[68000]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-02T05:20:01.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:01.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:01 smithi086 ceph-mon[88839]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-02T05:20:01.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:01.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:01 smithi086 ceph-mon[86443]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-02T05:20:01.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:03.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:03 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:20:03.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:03 smithi192 ceph-mon[68000]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-02T05:20:03.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:20:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:20:03] "GET /metrics HTTP/1.1" 200 38279 "" "Prometheus/2.43.0" 2024-04-02T05:20:03.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:03 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:20:03.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:03 smithi086 ceph-mon[88839]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-02T05:20:03.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:03 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:20:03.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:03 smithi086 ceph-mon[86443]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-02T05:20:05.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:05 smithi192 ceph-mon[68000]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:20:05.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:05 smithi086 ceph-mon[86443]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:20:05.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:05 smithi086 ceph-mon[88839]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:20:06.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:06 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-02T05:20:06.832 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:06 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-02T05:20:06.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:06 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-02T05:20:07.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:07 smithi192 ceph-mon[68000]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-02T05:20:07.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:07 smithi192 ceph-mon[68000]: Upgrade: osd.7 is safe to restart 2024-04-02T05:20:07.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:07 smithi192 ceph-mon[68000]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:20:07.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:07 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:07.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:07 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-02T05:20:07.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:07 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:20:07.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:07 smithi086 ceph-mon[86443]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-02T05:20:07.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:07 smithi086 ceph-mon[86443]: Upgrade: osd.7 is safe to restart 2024-04-02T05:20:07.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:07 smithi086 ceph-mon[86443]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:20:07.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:07 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:07.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:07 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-02T05:20:07.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:07 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:20:07.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:07 smithi086 ceph-mon[88839]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-02T05:20:07.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:07 smithi086 ceph-mon[88839]: Upgrade: osd.7 is safe to restart 2024-04-02T05:20:07.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:07 smithi086 ceph-mon[88839]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:20:07.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:07 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:07.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:07 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-02T05:20:07.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:07 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:20:08.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:08 smithi192 ceph-mon[68000]: Upgrade: Updating osd.7 2024-04-02T05:20:08.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:08 smithi192 ceph-mon[68000]: Deploying daemon osd.7 on smithi192 2024-04-02T05:20:08.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:08 smithi086 ceph-mon[86443]: Upgrade: Updating osd.7 2024-04-02T05:20:08.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:08 smithi086 ceph-mon[86443]: Deploying daemon osd.7 on smithi192 2024-04-02T05:20:08.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:08 smithi086 ceph-mon[88839]: Upgrade: Updating osd.7 2024-04-02T05:20:08.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:08 smithi086 ceph-mon[88839]: Deploying daemon osd.7 on smithi192 2024-04-02T05:20:09.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:09 smithi192 ceph-mon[68000]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:20:09.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:09 smithi086 ceph-mon[86443]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:20:09.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:09 smithi086 ceph-mon[88839]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:20:10.645 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:10 smithi192 systemd[1]: Stopping Ceph osd.7 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:20:10.645 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:10 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:20:10.645 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:10 smithi192 ceph-mon[68000]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:20:10.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:10 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:20:10.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:10 smithi086 ceph-mon[86443]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:20:10.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:10 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:20:10.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:10 smithi086 ceph-mon[88839]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:20:10.927 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[37891]: 2024-04-02T05:20:10.641+0000 7ff4ff106700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T05:20:10.927 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[37891]: 2024-04-02T05:20:10.641+0000 7ff4ff106700 -1 osd.7 152 *** Got signal Terminated *** 2024-04-02T05:20:10.928 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:10 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[37891]: 2024-04-02T05:20:10.641+0000 7ff4ff106700 -1 osd.7 152 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T05:20:11.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:11 smithi192 ceph-mon[68000]: osd.7 marked itself down and dead 2024-04-02T05:20:11.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:11 smithi086 ceph-mon[86443]: osd.7 marked itself down and dead 2024-04-02T05:20:11.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:11 smithi086 ceph-mon[88839]: osd.7 marked itself down and dead 2024-04-02T05:20:12.262 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:11 smithi192 podman[91858]: 2024-04-02 05:20:11.989821753 +0000 UTC m=+1.450075221 container died 95f5a035399990a23f4ef99f88060776470988b3874f71c7a558d311eda1186e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7, io.openshift.expose-services=, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, ceph=True, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, RELEASE=HEAD, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., distribution-scope=public) 2024-04-02T05:20:12.629 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:12 smithi192 ceph-mon[68000]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-02T05:20:12.629 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:12 smithi192 ceph-mon[68000]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-04-02T05:20:12.629 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:12 smithi192 ceph-mon[68000]: osdmap e153: 8 total, 7 up, 8 in 2024-04-02T05:20:12.629 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:12 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:20:12.629 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:12 smithi192 ceph-mon[68000]: pgmap v226: 161 pgs: 13 peering, 18 stale+active+clean, 130 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:20:12.629 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:12 smithi192 podman[91858]: 2024-04-02 05:20:12.359246853 +0000 UTC m=+1.819500319 container cleanup 95f5a035399990a23f4ef99f88060776470988b3874f71c7a558d311eda1186e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, version=8, ceph=True, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., release=754, maintainer=Guillaume Abrioux , name=centos-stream, GIT_CLEAN=True, com.redhat.component=centos-stream-container, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD) 2024-04-02T05:20:12.629 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:12 smithi192 bash[91858]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7 2024-04-02T05:20:12.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:12 smithi086 ceph-mon[86443]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-02T05:20:12.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:12 smithi086 ceph-mon[86443]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-04-02T05:20:12.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:12 smithi086 ceph-mon[86443]: osdmap e153: 8 total, 7 up, 8 in 2024-04-02T05:20:12.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:12 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:20:12.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:12 smithi086 ceph-mon[86443]: pgmap v226: 161 pgs: 13 peering, 18 stale+active+clean, 130 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:20:12.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:12 smithi086 ceph-mon[88839]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-02T05:20:12.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:12 smithi086 ceph-mon[88839]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-04-02T05:20:12.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:12 smithi086 ceph-mon[88839]: osdmap e153: 8 total, 7 up, 8 in 2024-04-02T05:20:12.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:12 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:20:12.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:12 smithi086 ceph-mon[88839]: pgmap v226: 161 pgs: 13 peering, 18 stale+active+clean, 130 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:20:12.927 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:12 smithi192 podman[91875]: 2024-04-02 05:20:12.626981317 +0000 UTC m=+0.640947140 container remove 95f5a035399990a23f4ef99f88060776470988b3874f71c7a558d311eda1186e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7, maintainer=Guillaume Abrioux , io.openshift.expose-services=, architecture=x86_64, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, ceph=True, RELEASE=HEAD, name=centos-stream, version=8, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-02T05:20:13.241 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:12 smithi192 podman[91971]: 2024-04-02 05:20:12.849863573 +0000 UTC m=+0.017834731 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:20:13.241 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:12 smithi192 podman[91971]: 2024-04-02 05:20:12.952906788 +0000 UTC m=+0.120877934 container create d95df498acc62a2ec788d93b8daf436627072aaf3021b9e6dd529ce8ceb4c2c3 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7-deactivate, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-6cec793, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326) 2024-04-02T05:20:13.241 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:13 smithi192 podman[91971]: 2024-04-02 05:20:13.164050908 +0000 UTC m=+0.332022063 container init d95df498acc62a2ec788d93b8daf436627072aaf3021b9e6dd529ce8ceb4c2c3 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-6cec793, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-04-02T05:20:13.241 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:13 smithi192 podman[91971]: 2024-04-02 05:20:13.167387029 +0000 UTC m=+0.335358177 container start d95df498acc62a2ec788d93b8daf436627072aaf3021b9e6dd529ce8ceb4c2c3 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-6cec793, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-04-02T05:20:13.241 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:13 smithi192 podman[91971]: 2024-04-02 05:20:13.238791919 +0000 UTC m=+0.406763072 container attach d95df498acc62a2ec788d93b8daf436627072aaf3021b9e6dd529ce8ceb4c2c3 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-6cec793, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326) 2024-04-02T05:20:13.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:13 smithi192 ceph-mon[68000]: Health check failed: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2024-04-02T05:20:13.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:13 smithi192 ceph-mon[68000]: osdmap e154: 8 total, 7 up, 8 in 2024-04-02T05:20:13.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:13 smithi192 podman[91971]: 2024-04-02 05:20:13.458965114 +0000 UTC m=+0.626936256 container died d95df498acc62a2ec788d93b8daf436627072aaf3021b9e6dd529ce8ceb4c2c3 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7-deactivate, org.label-schema.schema-version=1.0, RELEASE=squid-6cec793, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-04-02T05:20:13.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:13 smithi086 ceph-mon[86443]: Health check failed: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2024-04-02T05:20:13.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:13 smithi086 ceph-mon[86443]: osdmap e154: 8 total, 7 up, 8 in 2024-04-02T05:20:13.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:20:13 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:20:13] "GET /metrics HTTP/1.1" 200 38281 "" "Prometheus/2.43.0" 2024-04-02T05:20:13.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:13 smithi086 ceph-mon[88839]: Health check failed: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2024-04-02T05:20:13.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:13 smithi086 ceph-mon[88839]: osdmap e154: 8 total, 7 up, 8 in 2024-04-02T05:20:14.018 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:14 smithi192 podman[91997]: 2024-04-02 05:20:14.001162279 +0000 UTC m=+0.537504626 container remove d95df498acc62a2ec788d93b8daf436627072aaf3021b9e6dd529ce8ceb4c2c3 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-6cec793, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326) 2024-04-02T05:20:14.414 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:14 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.7.service: Deactivated successfully. 2024-04-02T05:20:14.414 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:14 smithi192 systemd[1]: Stopped Ceph osd.7 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:20:14.414 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:14 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.7.service: Consumed 17.431s CPU time. 2024-04-02T05:20:14.414 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:14 smithi192 systemd[1]: Starting Ceph osd.7 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:20:14.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:14 smithi192 ceph-mon[68000]: pgmap v228: 161 pgs: 5 active+undersized+degraded, 26 peering, 12 stale+active+clean, 8 active+undersized, 110 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 18/723 objects degraded (2.490%) 2024-04-02T05:20:14.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:14 smithi192 podman[92106]: 2024-04-02 05:20:14.332726232 +0000 UTC m=+0.018219918 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:20:14.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:14 smithi192 podman[92106]: 2024-04-02 05:20:14.444511237 +0000 UTC m=+0.130004909 container create 7f33072d361f20fd20c2b08a5fc063025b45081d02ab7b710dc336076bb45214 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7-activate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=squid-6cec793, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-02T05:20:14.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:14 smithi192 podman[92106]: 2024-04-02 05:20:14.638618267 +0000 UTC m=+0.324111952 container init 7f33072d361f20fd20c2b08a5fc063025b45081d02ab7b710dc336076bb45214 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7-activate, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-6cec793, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-04-02T05:20:14.678 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:14 smithi192 podman[92106]: 2024-04-02 05:20:14.641811447 +0000 UTC m=+0.327305121 container start 7f33072d361f20fd20c2b08a5fc063025b45081d02ab7b710dc336076bb45214 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=squid-6cec793, org.label-schema.build-date=20240326, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-04-02T05:20:14.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:14 smithi086 ceph-mon[88839]: pgmap v228: 161 pgs: 5 active+undersized+degraded, 26 peering, 12 stale+active+clean, 8 active+undersized, 110 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 18/723 objects degraded (2.490%) 2024-04-02T05:20:14.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:14 smithi086 ceph-mon[86443]: pgmap v228: 161 pgs: 5 active+undersized+degraded, 26 peering, 12 stale+active+clean, 8 active+undersized, 110 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 18/723 objects degraded (2.490%) 2024-04-02T05:20:15.084 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:14 smithi192 podman[92106]: 2024-04-02 05:20:14.713865617 +0000 UTC m=+0.399359303 container attach 7f33072d361f20fd20c2b08a5fc063025b45081d02ab7b710dc336076bb45214 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7-activate, org.label-schema.vendor=CentOS, RELEASE=squid-6cec793, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-04-02T05:20:15.415 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:15 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7-activate[92121]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-02T05:20:15.415 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:15 smithi192 bash[92106]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-02T05:20:15.415 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:15 smithi192 bash[92106]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-02T05:20:15.415 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:15 smithi192 bash[92106]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-02T05:20:15.416 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:15 smithi192 bash[92106]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-02T05:20:15.416 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:15 smithi192 bash[92106]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-04-02T05:20:15.416 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:15 smithi192 bash[92106]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-02T05:20:15.416 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:15 smithi192 bash[92106]: --> ceph-volume raw activate successful for osd ID: 7 2024-04-02T05:20:15.416 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:15 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7-activate[92121]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-02T05:20:15.416 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:15 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7-activate[92121]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-02T05:20:15.416 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:15 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7-activate[92121]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-02T05:20:15.417 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:15 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7-activate[92121]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-04-02T05:20:15.417 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:15 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7-activate[92121]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-02T05:20:15.417 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:15 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7-activate[92121]: --> ceph-volume raw activate successful for osd ID: 7 2024-04-02T05:20:15.417 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:15 smithi192 podman[92106]: 2024-04-02 05:20:15.107020888 +0000 UTC m=+0.792514573 container died 7f33072d361f20fd20c2b08a5fc063025b45081d02ab7b710dc336076bb45214 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, ceph=True, RELEASE=squid-6cec793, CEPH_POINT_RELEASE=) 2024-04-02T05:20:15.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:15 smithi192 ceph-mon[68000]: Health check failed: Degraded data redundancy: 18/723 objects degraded (2.490%), 5 pgs degraded (PG_DEGRADED) 2024-04-02T05:20:15.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:15 smithi086 ceph-mon[88839]: Health check failed: Degraded data redundancy: 18/723 objects degraded (2.490%), 5 pgs degraded (PG_DEGRADED) 2024-04-02T05:20:15.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:15 smithi086 ceph-mon[86443]: Health check failed: Degraded data redundancy: 18/723 objects degraded (2.490%), 5 pgs degraded (PG_DEGRADED) 2024-04-02T05:20:15.979 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:15 smithi192 podman[92243]: 2024-04-02 05:20:15.701582048 +0000 UTC m=+0.589133595 container remove 7f33072d361f20fd20c2b08a5fc063025b45081d02ab7b710dc336076bb45214 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7-activate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-6cec793, CEPH_POINT_RELEASE=) 2024-04-02T05:20:15.979 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:15 smithi192 podman[92310]: 2024-04-02 05:20:15.817129213 +0000 UTC m=+0.018210150 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:20:16.230 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:15 smithi192 podman[92310]: 2024-04-02 05:20:15.977934828 +0000 UTC m=+0.179015742 container create 48f6eb4f415f24c4455e4cf4eecbe54f271944b84df8bd13d5bfc211711232cb (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, GIT_CLEAN=True, RELEASE=squid-6cec793, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-04-02T05:20:16.230 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:16 smithi192 podman[92310]: 2024-04-02 05:20:16.205377998 +0000 UTC m=+0.406458912 container init 48f6eb4f415f24c4455e4cf4eecbe54f271944b84df8bd13d5bfc211711232cb (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-6cec793, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-04-02T05:20:16.230 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:16 smithi192 podman[92310]: 2024-04-02 05:20:16.208279585 +0000 UTC m=+0.409360507 container start 48f6eb4f415f24c4455e4cf4eecbe54f271944b84df8bd13d5bfc211711232cb (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-6cec793, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-04-02T05:20:16.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:16 smithi192 ceph-osd[92329]: -- 172.21.15.192:0/2564666626 <== mon.2 v2:172.21.15.192:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d696e29680 con 0x55d696df2c00 2024-04-02T05:20:16.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:16 smithi192 bash[92310]: 48f6eb4f415f24c4455e4cf4eecbe54f271944b84df8bd13d5bfc211711232cb 2024-04-02T05:20:16.678 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:16 smithi192 systemd[1]: Started Ceph osd.7 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:20:17.177 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:16 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:20:16.746+0000 7fd45b3a17c0 -1 Falling back to public interface 2024-04-02T05:20:17.595 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:17 smithi192 ceph-mon[68000]: pgmap v229: 161 pgs: 15 active+undersized+degraded, 26 peering, 26 active+undersized, 94 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 56/723 objects degraded (7.746%) 2024-04-02T05:20:17.595 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:17 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:17.595 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:17 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:17.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:17 smithi086 ceph-mon[86443]: pgmap v229: 161 pgs: 15 active+undersized+degraded, 26 peering, 26 active+undersized, 94 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 56/723 objects degraded (7.746%) 2024-04-02T05:20:17.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:17 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:17.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:17 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:17.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:17 smithi086 ceph-mon[88839]: pgmap v229: 161 pgs: 15 active+undersized+degraded, 26 peering, 26 active+undersized, 94 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 56/723 objects degraded (7.746%) 2024-04-02T05:20:17.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:17 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:17.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:17 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:18.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:18 smithi192 ceph-mon[68000]: pgmap v230: 161 pgs: 26 active+undersized+degraded, 41 active+undersized, 94 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 90/723 objects degraded (12.448%) 2024-04-02T05:20:18.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:18 smithi192 ceph-mon[68000]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 4 pgs peering) 2024-04-02T05:20:18.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:18 smithi086 ceph-mon[86443]: pgmap v230: 161 pgs: 26 active+undersized+degraded, 41 active+undersized, 94 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 90/723 objects degraded (12.448%) 2024-04-02T05:20:18.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:18 smithi086 ceph-mon[86443]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 4 pgs peering) 2024-04-02T05:20:18.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:18 smithi086 ceph-mon[88839]: pgmap v230: 161 pgs: 26 active+undersized+degraded, 41 active+undersized, 94 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 90/723 objects degraded (12.448%) 2024-04-02T05:20:18.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:18 smithi086 ceph-mon[88839]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 4 pgs peering) 2024-04-02T05:20:19.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:19 smithi086 ceph-mon[88839]: from='osd.7 [v2:172.21.15.192:6824/1466955259,v1:172.21.15.192:6825/1466955259]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-02T05:20:19.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:19 smithi086 ceph-mon[88839]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-02T05:20:19.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:19 smithi086 ceph-mon[86443]: from='osd.7 [v2:172.21.15.192:6824/1466955259,v1:172.21.15.192:6825/1466955259]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-02T05:20:19.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:19 smithi086 ceph-mon[86443]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-02T05:20:19.848 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:19 smithi192 ceph-mon[68000]: from='osd.7 [v2:172.21.15.192:6824/1466955259,v1:172.21.15.192:6825/1466955259]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-02T05:20:19.849 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:19 smithi192 ceph-mon[68000]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-02T05:20:19.849 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:20:19.475+0000 7fd45b3a17c0 -1 osd.7 0 read_superblock omap replica is missing. 2024-04-02T05:20:19.849 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:19 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:20:19.509+0000 7fd45b3a17c0 -1 osd.7 152 log_to_monitors true 2024-04-02T05:20:20.177 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:20:20 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:20:20.036+0000 7fd4487f1700 -1 osd.7 152 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T05:20:20.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:20 smithi192 ceph-mon[68000]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-02T05:20:20.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:20 smithi192 ceph-mon[68000]: osdmap e155: 8 total, 7 up, 8 in 2024-04-02T05:20:20.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:20 smithi192 ceph-mon[68000]: from='osd.7 [v2:172.21.15.192:6824/1466955259,v1:172.21.15.192:6825/1466955259]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T05:20:20.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:20 smithi192 ceph-mon[68000]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T05:20:20.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:20 smithi192 ceph-mon[68000]: from='osd.7 ' entity='osd.7' 2024-04-02T05:20:20.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:20 smithi192 ceph-mon[68000]: pgmap v232: 161 pgs: 26 active+undersized+degraded, 41 active+undersized, 94 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s; 90/723 objects degraded (12.448%) 2024-04-02T05:20:20.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:20 smithi086 ceph-mon[86443]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-02T05:20:20.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:20 smithi086 ceph-mon[86443]: osdmap e155: 8 total, 7 up, 8 in 2024-04-02T05:20:20.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:20 smithi086 ceph-mon[86443]: from='osd.7 [v2:172.21.15.192:6824/1466955259,v1:172.21.15.192:6825/1466955259]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T05:20:20.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:20 smithi086 ceph-mon[86443]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T05:20:20.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:20 smithi086 ceph-mon[86443]: from='osd.7 ' entity='osd.7' 2024-04-02T05:20:20.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:20 smithi086 ceph-mon[86443]: pgmap v232: 161 pgs: 26 active+undersized+degraded, 41 active+undersized, 94 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s; 90/723 objects degraded (12.448%) 2024-04-02T05:20:20.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:20 smithi086 ceph-mon[88839]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-02T05:20:20.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:20 smithi086 ceph-mon[88839]: osdmap e155: 8 total, 7 up, 8 in 2024-04-02T05:20:20.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:20 smithi086 ceph-mon[88839]: from='osd.7 [v2:172.21.15.192:6824/1466955259,v1:172.21.15.192:6825/1466955259]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T05:20:20.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:20 smithi086 ceph-mon[88839]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi192", "root=default"]}]: dispatch 2024-04-02T05:20:20.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:20 smithi086 ceph-mon[88839]: from='osd.7 ' entity='osd.7' 2024-04-02T05:20:20.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:20 smithi086 ceph-mon[88839]: pgmap v232: 161 pgs: 26 active+undersized+degraded, 41 active+undersized, 94 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s; 90/723 objects degraded (12.448%) 2024-04-02T05:20:22.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:21 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:22.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:21 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:22.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:21 smithi086 ceph-mon[86443]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-02T05:20:22.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:21 smithi086 ceph-mon[86443]: osd.7 [v2:172.21.15.192:6824/1466955259,v1:172.21.15.192:6825/1466955259] boot 2024-04-02T05:20:22.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:21 smithi086 ceph-mon[86443]: osdmap e156: 8 total, 8 up, 8 in 2024-04-02T05:20:22.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:21 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T05:20:22.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:21 smithi086 ceph-mon[86443]: Health check update: Degraded data redundancy: 90/723 objects degraded (12.448%), 26 pgs degraded (PG_DEGRADED) 2024-04-02T05:20:22.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:21 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:22.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:21 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:22.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:21 smithi086 ceph-mon[88839]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-02T05:20:22.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:21 smithi086 ceph-mon[88839]: osd.7 [v2:172.21.15.192:6824/1466955259,v1:172.21.15.192:6825/1466955259] boot 2024-04-02T05:20:22.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:21 smithi086 ceph-mon[88839]: osdmap e156: 8 total, 8 up, 8 in 2024-04-02T05:20:22.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:21 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T05:20:22.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:21 smithi086 ceph-mon[88839]: Health check update: Degraded data redundancy: 90/723 objects degraded (12.448%), 26 pgs degraded (PG_DEGRADED) 2024-04-02T05:20:22.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:21 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:22.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:21 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:22.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:21 smithi192 ceph-mon[68000]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-02T05:20:22.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:21 smithi192 ceph-mon[68000]: osd.7 [v2:172.21.15.192:6824/1466955259,v1:172.21.15.192:6825/1466955259] boot 2024-04-02T05:20:22.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:21 smithi192 ceph-mon[68000]: osdmap e156: 8 total, 8 up, 8 in 2024-04-02T05:20:22.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:21 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T05:20:22.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:21 smithi192 ceph-mon[68000]: Health check update: Degraded data redundancy: 90/723 objects degraded (12.448%), 26 pgs degraded (PG_DEGRADED) 2024-04-02T05:20:23.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:22 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:20:23.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:22 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:23.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:22 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:23.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:22 smithi192 ceph-mon[68000]: osdmap e157: 8 total, 8 up, 8 in 2024-04-02T05:20:23.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:22 smithi192 ceph-mon[68000]: pgmap v235: 161 pgs: 3 peering, 20 active+undersized+degraded, 34 active+undersized, 104 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s; 73/723 objects degraded (10.097%) 2024-04-02T05:20:23.235 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:22 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:20:23.235 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:22 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:23.235 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:22 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:23.235 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:22 smithi086 ceph-mon[86443]: osdmap e157: 8 total, 8 up, 8 in 2024-04-02T05:20:23.235 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:22 smithi086 ceph-mon[86443]: pgmap v235: 161 pgs: 3 peering, 20 active+undersized+degraded, 34 active+undersized, 104 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s; 73/723 objects degraded (10.097%) 2024-04-02T05:20:23.235 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:22 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:20:23.235 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:22 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:23.235 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:22 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:23.236 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:22 smithi086 ceph-mon[88839]: osdmap e157: 8 total, 8 up, 8 in 2024-04-02T05:20:23.236 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:22 smithi086 ceph-mon[88839]: pgmap v235: 161 pgs: 3 peering, 20 active+undersized+degraded, 34 active+undersized, 104 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s; 73/723 objects degraded (10.097%) 2024-04-02T05:20:23.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:20:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:20:23] "GET /metrics HTTP/1.1" 200 38281 "" "Prometheus/2.43.0" 2024-04-02T05:20:24.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:24 smithi086 ceph-mon[86443]: pgmap v236: 161 pgs: 3 peering, 12 active+undersized+degraded, 22 active+undersized, 124 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 44/723 objects degraded (6.086%) 2024-04-02T05:20:24.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:24 smithi086 ceph-mon[88839]: pgmap v236: 161 pgs: 3 peering, 12 active+undersized+degraded, 22 active+undersized, 124 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 44/723 objects degraded (6.086%) 2024-04-02T05:20:24.719 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:24 smithi192 ceph-mon[68000]: pgmap v236: 161 pgs: 3 peering, 12 active+undersized+degraded, 22 active+undersized, 124 active+clean; 585 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 44/723 objects degraded (6.086%) 2024-04-02T05:20:26.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:26.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:20:26.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:26.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:26.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:20:26.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:20:26.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:26.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:20:26.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:26.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:26.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:26.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:26.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:26.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-04-02T05:20:26.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-04-02T05:20:26.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-04-02T05:20:26.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-04-02T05:20:26.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-04-02T05:20:26.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-04-02T05:20:26.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-04-02T05:20:26.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-04-02T05:20:26.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-04-02T05:20:26.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-04-02T05:20:26.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-04-02T05:20:26.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-04-02T05:20:26.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-04-02T05:20:26.585 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-04-02T05:20:26.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:26.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:20:26.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:26.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:26.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:20:26.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:20:26.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:26.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:20:26.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:26.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:26.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:26.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:26.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:26.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-04-02T05:20:26.587 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-04-02T05:20:26.587 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-04-02T05:20:26.587 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-04-02T05:20:26.587 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-04-02T05:20:26.587 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-04-02T05:20:26.587 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-04-02T05:20:26.587 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-04-02T05:20:26.587 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-04-02T05:20:26.587 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-04-02T05:20:26.587 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-04-02T05:20:26.587 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-04-02T05:20:26.587 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-04-02T05:20:26.587 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-04-02T05:20:26.588 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-04-02T05:20:26.588 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-04-02T05:20:26.588 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-04-02T05:20:26.588 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-04-02T05:20:26.588 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-04-02T05:20:26.588 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-04-02T05:20:26.588 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-04-02T05:20:26.588 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-04-02T05:20:26.588 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-04-02T05:20:26.588 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-04-02T05:20:26.588 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-04-02T05:20:26.588 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-04-02T05:20:26.589 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-04-02T05:20:26.589 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-04-02T05:20:26.589 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-04-02T05:20:26.589 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-04-02T05:20:26.589 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-04-02T05:20:26.589 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-04-02T05:20:26.589 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-04-02T05:20:26.589 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-04-02T05:20:26.589 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-04-02T05:20:26.589 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-04-02T05:20:26.589 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-04-02T05:20:26.589 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:26 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-04-02T05:20:26.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:26.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:20:26.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:26.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:26.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:20:26.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:20:26.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:26.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:20:26.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:26.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:26.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:26.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:26.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:26.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-04-02T05:20:26.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-04-02T05:20:26.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-04-02T05:20:26.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-04-02T05:20:26.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-04-02T05:20:26.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-04-02T05:20:26.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-04-02T05:20:26.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-04-02T05:20:26.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-04-02T05:20:26.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-04-02T05:20:26.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-04-02T05:20:26.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-04-02T05:20:26.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-04-02T05:20:26.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-04-02T05:20:26.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-04-02T05:20:26.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-04-02T05:20:26.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-04-02T05:20:26.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-04-02T05:20:26.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-04-02T05:20:26.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-04-02T05:20:26.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-04-02T05:20:26.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-04-02T05:20:26.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-04-02T05:20:26.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-04-02T05:20:26.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-04-02T05:20:26.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:26 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-04-02T05:20:27.432 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all osd 2024-04-02T05:20:27.432 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: Upgrade: Setting require_osd_release to 19 squid 2024-04-02T05:20:27.432 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 754 B/s rd, 0 op/s 2024-04-02T05:20:27.432 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-04-02T05:20:27.432 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-04-02T05:20:27.432 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: osdmap e158: 8 total, 8 up, 8 in 2024-04-02T05:20:27.432 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:27.432 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:27.432 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:27.432 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:27.432 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:27.432 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:27.432 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:27.433 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:27.433 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:27.433 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:27.433 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:27.433 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:27.433 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:27.433 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:27.433 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:27.433 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:27.433 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:27.433 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:20:27.433 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:20:27.433 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-02T05:20:27.433 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.433 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.434 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-02T05:20:27.434 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:20:27.434 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all osd 2024-04-02T05:20:27.434 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: Upgrade: Setting require_osd_release to 19 squid 2024-04-02T05:20:27.434 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 754 B/s rd, 0 op/s 2024-04-02T05:20:27.434 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-04-02T05:20:27.434 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-04-02T05:20:27.434 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: osdmap e158: 8 total, 8 up, 8 in 2024-04-02T05:20:27.434 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:27.434 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:27.435 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:27.435 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:27.435 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:27.435 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:27.435 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:27.435 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:27.435 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:27.435 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:27.435 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:27.435 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:27.435 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:27.435 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:27.435 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:27.435 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:27.435 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:27.435 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:20:27.436 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:20:27.436 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-02T05:20:27.436 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.436 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.436 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-02T05:20:27.436 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:20:27.436 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:20:27.436 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-02T05:20:27.437 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:20:27.437 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:20:27.437 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-02T05:20:27.437 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:20:27.437 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:20:27.437 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: Health check update: Degraded data redundancy: 44/723 objects degraded (6.086%), 12 pgs degraded (PG_DEGRADED) 2024-04-02T05:20:27.437 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-02T05:20:27.437 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:20:27.437 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:20:27.437 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:20:27.437 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:20:27.437 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-02T05:20:27.437 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.438 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.438 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:20:27.438 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:20:27.438 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-02T05:20:27.438 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:20:27.438 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:20:27.438 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:20:27.438 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:20:27.438 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-02T05:20:27.438 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:20:27.438 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:20:27.438 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-02T05:20:27.438 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.438 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.439 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.439 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.439 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.439 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.439 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.439 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.439 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.439 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.439 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.439 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.439 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:20:27.439 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:20:27.439 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-02T05:20:27.439 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:20:27.439 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:20:27.440 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:27.440 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:20:27.440 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:20:27.440 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:27.440 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:20:27.440 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-02T05:20:27.440 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:20:27.440 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:20:27.440 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-02T05:20:27.440 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:20:27.440 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:20:27.441 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: Health check update: Degraded data redundancy: 44/723 objects degraded (6.086%), 12 pgs degraded (PG_DEGRADED) 2024-04-02T05:20:27.441 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-02T05:20:27.441 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:20:27.441 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:20:27.441 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:20:27.441 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:20:27.441 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-02T05:20:27.441 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.441 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.441 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:20:27.441 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:20:27.441 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-02T05:20:27.441 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:20:27.441 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:20:27.441 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:20:27.441 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:20:27.442 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-02T05:20:27.442 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:20:27.442 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:20:27.442 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-02T05:20:27.442 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.442 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.442 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.442 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.442 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.442 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.442 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.442 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.442 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.442 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.443 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.443 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.443 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:20:27.443 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:20:27.443 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-02T05:20:27.443 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:20:27.443 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:20:27.443 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:27.443 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:20:27.443 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:20:27.443 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:27 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:27.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all osd 2024-04-02T05:20:27.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: Upgrade: Setting require_osd_release to 19 squid 2024-04-02T05:20:27.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 754 B/s rd, 0 op/s 2024-04-02T05:20:27.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-04-02T05:20:27.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-04-02T05:20:27.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: osdmap e158: 8 total, 8 up, 8 in 2024-04-02T05:20:27.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:27.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:27.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:27.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:27.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:27.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:27.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:27.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:27.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:27.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:27.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:27.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:27.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:27.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:27.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:27.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:27.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:27.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:20:27.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:20:27.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-02T05:20:27.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-02T05:20:27.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:20:27.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:20:27.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-02T05:20:27.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:20:27.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:20:27.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-02T05:20:27.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:20:27.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:20:27.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: Health check update: Degraded data redundancy: 44/723 objects degraded (6.086%), 12 pgs degraded (PG_DEGRADED) 2024-04-02T05:20:27.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-02T05:20:27.681 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:20:27.681 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:20:27.681 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:20:27.681 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:20:27.681 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-02T05:20:27.681 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.681 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.681 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:20:27.681 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:20:27.681 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-02T05:20:27.681 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:20:27.681 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:20:27.681 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:20:27.681 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:20:27.681 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-02T05:20:27.682 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:20:27.682 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:20:27.682 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-02T05:20:27.682 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.682 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.682 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.682 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.682 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.682 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.682 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.682 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.682 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.682 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.683 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.683 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:20:27.683 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:20:27.683 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:20:27.683 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-02T05:20:27.683 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:20:27.683 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:20:27.683 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:27.683 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:20:27.683 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:20:27.683 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:27 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:27.916 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch ps' 2024-04-02T05:20:28.486 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:28 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all mds 2024-04-02T05:20:28.486 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:28 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all rbd-mirror 2024-04-02T05:20:28.486 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:28 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all ceph-exporter 2024-04-02T05:20:28.486 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:28 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all nfs 2024-04-02T05:20:28.486 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:28 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all nvmeof 2024-04-02T05:20:28.486 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:28 smithi086 ceph-mon[86443]: Upgrade: Finalizing container_image settings 2024-04-02T05:20:28.486 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:28 smithi086 ceph-mon[86443]: Upgrade: Complete! 2024-04-02T05:20:28.487 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:28 smithi086 ceph-mon[86443]: from='client.64516 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:20:28.487 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:28 smithi086 ceph-mon[86443]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 44/723 objects degraded (6.086%), 12 pgs degraded) 2024-04-02T05:20:28.487 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:28 smithi086 ceph-mon[86443]: Cluster is now healthy 2024-04-02T05:20:28.487 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:28 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all mds 2024-04-02T05:20:28.487 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:28 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all rbd-mirror 2024-04-02T05:20:28.487 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:28 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all ceph-exporter 2024-04-02T05:20:28.487 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:28 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all nfs 2024-04-02T05:20:28.487 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:28 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all nvmeof 2024-04-02T05:20:28.487 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:28 smithi086 ceph-mon[88839]: Upgrade: Finalizing container_image settings 2024-04-02T05:20:28.487 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:28 smithi086 ceph-mon[88839]: Upgrade: Complete! 2024-04-02T05:20:28.487 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:28 smithi086 ceph-mon[88839]: from='client.64516 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:20:28.487 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:28 smithi086 ceph-mon[88839]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 44/723 objects degraded (6.086%), 12 pgs degraded) 2024-04-02T05:20:28.487 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:28 smithi086 ceph-mon[88839]: Cluster is now healthy 2024-04-02T05:20:28.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:28 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all mds 2024-04-02T05:20:28.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:28 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all rbd-mirror 2024-04-02T05:20:28.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:28 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all ceph-exporter 2024-04-02T05:20:28.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:28 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all nfs 2024-04-02T05:20:28.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:28 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all nvmeof 2024-04-02T05:20:28.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:28 smithi192 ceph-mon[68000]: Upgrade: Finalizing container_image settings 2024-04-02T05:20:28.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:28 smithi192 ceph-mon[68000]: Upgrade: Complete! 2024-04-02T05:20:28.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:28 smithi192 ceph-mon[68000]: from='client.64516 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:20:28.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:28 smithi192 ceph-mon[68000]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 44/723 objects degraded (6.086%), 12 pgs degraded) 2024-04-02T05:20:28.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:28 smithi192 ceph-mon[68000]: Cluster is now healthy 2024-04-02T05:20:29.502 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:29 smithi086 ceph-mon[88839]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-02T05:20:29.502 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:29 smithi086 ceph-mon[86443]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-02T05:20:29.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:29 smithi192 ceph-mon[68000]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-02T05:20:29.916 INFO:teuthology.orchestra.run.smithi086.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T05:20:29.916 INFO:teuthology.orchestra.run.smithi086.stdout:alertmanager.a smithi086 *:9093,9094 running (8m) 2m ago 30m 26.1M - 0.25.0 c8568f914cd2 660ea62bb6b9 2024-04-02T05:20:29.916 INFO:teuthology.orchestra.run.smithi086.stdout:grafana.a smithi192 *:3000 running (8m) 8s ago 30m 79.0M - 9.4.12 2bacad6d85d8 259af9e63aa5 2024-04-02T05:20:29.916 INFO:teuthology.orchestra.run.smithi086.stdout:iscsi.foo.smithi086.rsbqwx smithi086 running (8m) 2m ago 29m 62.1M - 3.5 e1d6a67b021e fe728eb940f7 2024-04-02T05:20:29.916 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.x smithi192 *:8443,9283,8765 running (9m) 8s ago 34m 454M - 19.0.0-1979-g6cec793d 78f830190839 dd1e6d0f0a95 2024-04-02T05:20:29.916 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.y smithi086 *:8443,9283,8765 running (19m) 2m ago 36m 507M - 19.0.0-1979-g6cec793d 78f830190839 3dca54a018ab 2024-04-02T05:20:29.916 INFO:teuthology.orchestra.run.smithi086.stdout:mon.a smithi086 running (6m) 2m ago 36m 51.7M 2048M 19.0.0-1979-g6cec793d 78f830190839 8e478dd4da2d 2024-04-02T05:20:29.916 INFO:teuthology.orchestra.run.smithi086.stdout:mon.b smithi192 running (7m) 8s ago 35m 51.3M 2048M 19.0.0-1979-g6cec793d 78f830190839 4125e997188f 2024-04-02T05:20:29.916 INFO:teuthology.orchestra.run.smithi086.stdout:mon.c smithi086 running (6m) 2m ago 35m 42.9M 2048M 19.0.0-1979-g6cec793d 78f830190839 8180999b3142 2024-04-02T05:20:29.916 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.a smithi086 *:9100 running (9m) 2m ago 31m 16.3M - 1.5.0 0da6a335fe13 05a844569d7b 2024-04-02T05:20:29.916 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.b smithi192 *:9100 running (9m) 8s ago 31m 16.7M - 1.5.0 0da6a335fe13 1a7d6788d58b 2024-04-02T05:20:29.916 INFO:teuthology.orchestra.run.smithi086.stdout:osd.0 smithi086 running (4m) 2m ago 34m 46.0M 2992M 19.0.0-1979-g6cec793d 78f830190839 e521a0ef7daf 2024-04-02T05:20:29.917 INFO:teuthology.orchestra.run.smithi086.stdout:osd.1 smithi086 running (4m) 2m ago 34m 53.1M 2992M 19.0.0-1979-g6cec793d 78f830190839 76fe72844771 2024-04-02T05:20:29.917 INFO:teuthology.orchestra.run.smithi086.stdout:osd.2 smithi086 running (3m) 2m ago 33m 48.9M 2992M 19.0.0-1979-g6cec793d 78f830190839 28594f089a8f 2024-04-02T05:20:29.917 INFO:teuthology.orchestra.run.smithi086.stdout:osd.3 smithi086 running (2m) 2m ago 33m 12.3M 2992M 19.0.0-1979-g6cec793d 78f830190839 6028f927dfd6 2024-04-02T05:20:29.917 INFO:teuthology.orchestra.run.smithi086.stdout:osd.4 smithi192 running (117s) 8s ago 33m 52.4M 3248M 19.0.0-1979-g6cec793d 78f830190839 f9c58b761b62 2024-04-02T05:20:29.917 INFO:teuthology.orchestra.run.smithi086.stdout:osd.5 smithi192 running (83s) 8s ago 32m 51.3M 3248M 19.0.0-1979-g6cec793d 78f830190839 813afd5cf75d 2024-04-02T05:20:29.917 INFO:teuthology.orchestra.run.smithi086.stdout:osd.6 smithi192 running (49s) 8s ago 32m 51.6M 3248M 19.0.0-1979-g6cec793d 78f830190839 d3707b413a43 2024-04-02T05:20:29.917 INFO:teuthology.orchestra.run.smithi086.stdout:osd.7 smithi192 running (14s) 8s ago 31m 15.1M 3248M 19.0.0-1979-g6cec793d 78f830190839 48f6eb4f415f 2024-04-02T05:20:29.917 INFO:teuthology.orchestra.run.smithi086.stdout:prometheus.a smithi192 *:9095 running (8m) 8s ago 31m 72.6M - 2.43.0 a07b618ecd1d 8378fcdb60e3 2024-04-02T05:20:29.917 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi086.bxzxlp smithi086 *:8000 running (29m) 2m ago 29m 80.0M - 17.2.0 e1d6a67b021e 6725785fbc53 2024-04-02T05:20:29.917 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi192.pmlbot smithi192 *:8000 running (29m) 8s ago 29m 80.3M - 17.2.0 e1d6a67b021e 5ae3638f8b1c 2024-04-02T05:20:30.704 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:30 smithi086 ceph-mon[88839]: from='client.54579 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:20:30.704 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:30 smithi086 ceph-mon[88839]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:20:30.705 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:30 smithi086 ceph-mon[86443]: from='client.54579 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:20:30.705 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:30 smithi086 ceph-mon[86443]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:20:30.706 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 1'"'"'' 2024-04-02T05:20:30.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:30 smithi192 ceph-mon[68000]: from='client.54579 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:20:30.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:30 smithi192 ceph-mon[68000]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:20:31.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:31 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:31.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:31 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:31.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:31 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:32.701 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:32 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:20:32.701 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:32 smithi086 ceph-mon[88839]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:20:32.702 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:32 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:20:32.702 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:32 smithi086 ceph-mon[86443]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:20:32.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:32 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:20:32.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:32 smithi192 ceph-mon[68000]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:20:33.021 INFO:teuthology.orchestra.run.smithi086.stdout:true 2024-04-02T05:20:33.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:20:33 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:20:33] "GET /metrics HTTP/1.1" 200 38376 "" "Prometheus/2.43.0" 2024-04-02T05:20:33.729 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph versions | jq -e '"'"'.osd | keys'"'"' | grep $sha1' 2024-04-02T05:20:33.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:33 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/1607700580' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:34.032 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:33 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/1607700580' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:34.032 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:33 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/1607700580' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:34.888 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:34 smithi086 ceph-mon[86443]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:20:34.888 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:34 smithi086 ceph-mon[88839]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:20:34.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:34 smithi192 ceph-mon[68000]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:20:35.872 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)" 2024-04-02T05:20:36.617 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch upgrade status' 2024-04-02T05:20:36.922 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:36 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:36.922 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:36 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/3457370190' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:36.922 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:36 smithi086 ceph-mon[86443]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:20:36.922 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:36 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:36.922 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:36 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/3457370190' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:36.922 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:36 smithi086 ceph-mon[88839]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:20:36.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:36 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:36.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:36 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/3457370190' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:36.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:36 smithi192 ceph-mon[68000]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T05:20:38.582 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:38 smithi086 ceph-mon[88839]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-02T05:20:38.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:38 smithi086 ceph-mon[86443]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-02T05:20:38.759 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:20:38.759 INFO:teuthology.orchestra.run.smithi086.stdout: "target_image": null, 2024-04-02T05:20:38.759 INFO:teuthology.orchestra.run.smithi086.stdout: "in_progress": false, 2024-04-02T05:20:38.759 INFO:teuthology.orchestra.run.smithi086.stdout: "which": "", 2024-04-02T05:20:38.759 INFO:teuthology.orchestra.run.smithi086.stdout: "services_complete": [], 2024-04-02T05:20:38.759 INFO:teuthology.orchestra.run.smithi086.stdout: "progress": null, 2024-04-02T05:20:38.759 INFO:teuthology.orchestra.run.smithi086.stdout: "message": "", 2024-04-02T05:20:38.760 INFO:teuthology.orchestra.run.smithi086.stdout: "is_paused": false 2024-04-02T05:20:38.760 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:20:38.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:38 smithi192 ceph-mon[68000]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-02T05:20:39.381 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph health detail' 2024-04-02T05:20:39.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:39 smithi086 ceph-mon[86443]: from='client.54597 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:20:39.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:39 smithi086 ceph-mon[88839]: from='client.54597 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:20:39.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:39 smithi192 ceph-mon[68000]: from='client.54597 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:20:41.380 INFO:teuthology.orchestra.run.smithi086.stdout:HEALTH_OK 2024-04-02T05:20:41.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:41 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:41.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:41 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:20:41.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:41 smithi086 ceph-mon[86443]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:20:41.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:41 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:41.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:41 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:20:41.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:41 smithi086 ceph-mon[88839]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:20:41.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:41 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:41.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:41 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:20:41.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:41 smithi192 ceph-mon[68000]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:20:42.021 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo' 2024-04-02T05:20:42.581 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:42 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/3792574792' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T05:20:42.582 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:42 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/3792574792' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T05:20:42.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:42 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/3792574792' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T05:20:43.582 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:43 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:20:43.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:43 smithi086 ceph-mon[88839]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:20:43.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:43 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:20:43.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:43 smithi086 ceph-mon[86443]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:20:43.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:20:43 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:20:43] "GET /metrics HTTP/1.1" 200 38372 "" "Prometheus/2.43.0" 2024-04-02T05:20:43.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:43 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:20:43.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:43 smithi192 ceph-mon[68000]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:20:44.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:44 smithi086 ceph-mon[88839]: from='client.64552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:20:44.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:44 smithi086 ceph-mon[88839]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:20:44.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:44 smithi086 ceph-mon[86443]: from='client.64552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:20:44.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:44 smithi086 ceph-mon[86443]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:20:44.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:44 smithi192 ceph-mon[68000]: from='client.64552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:20:44.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:44 smithi192 ceph-mon[68000]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:20:45.867 INFO:teuthology.orchestra.run.smithi086.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:20:46.639 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-02T05:20:46.869 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:46 smithi086 ceph-mon[86443]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:20:46.869 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:46 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:46.869 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:46 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:20:46.869 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:46 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:20:46.869 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:46 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:46.869 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:46 smithi086 ceph-mon[86443]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:20:46.869 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:46 smithi086 ceph-mon[86443]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 0 B/s wr, 20 op/s 2024-04-02T05:20:47.120 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:46 smithi086 ceph-mon[88839]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:20:47.120 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:46 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:47.120 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:46 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:20:47.120 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:46 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:20:47.120 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:46 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:47.120 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:46 smithi086 ceph-mon[88839]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:20:47.120 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:46 smithi086 ceph-mon[88839]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 0 B/s wr, 20 op/s 2024-04-02T05:20:47.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:46 smithi192 ceph-mon[68000]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:20:47.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:46 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:47.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:46 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:20:47.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:46 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:20:47.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:46 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:47.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:46 smithi192 ceph-mon[68000]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:20:47.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:46 smithi192 ceph-mon[68000]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 0 B/s wr, 20 op/s 2024-04-02T05:20:49.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:49.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[86443]: Upgrade: Target is version 19.0.0-1979-g6cec793d (squid) 2024-04-02T05:20:49.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[86443]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7bd5b84926d4a0d6c51e5562e965bff211cec2740c9f4dec9b5e00a2bbb75eae'] 2024-04-02T05:20:49.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:20:49.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:49.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all mgr 2024-04-02T05:20:49.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:49.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:49.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all mon 2024-04-02T05:20:49.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:49.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:49.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all crash 2024-04-02T05:20:49.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:49.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:49.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all osd 2024-04-02T05:20:49.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:49.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:49.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all mds 2024-04-02T05:20:49.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:49.084 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[86443]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 39 KiB/s rd, 0 B/s wr, 64 op/s 2024-04-02T05:20:49.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:49.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[88839]: Upgrade: Target is version 19.0.0-1979-g6cec793d (squid) 2024-04-02T05:20:49.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[88839]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7bd5b84926d4a0d6c51e5562e965bff211cec2740c9f4dec9b5e00a2bbb75eae'] 2024-04-02T05:20:49.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:20:49.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:49.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all mgr 2024-04-02T05:20:49.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:49.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:49.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all mon 2024-04-02T05:20:49.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:49.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:49.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all crash 2024-04-02T05:20:49.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:49.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:49.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all osd 2024-04-02T05:20:49.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:49.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:49.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all mds 2024-04-02T05:20:49.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:49.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:48 smithi086 ceph-mon[88839]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 39 KiB/s rd, 0 B/s wr, 64 op/s 2024-04-02T05:20:49.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:48 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:49.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:48 smithi192 ceph-mon[68000]: Upgrade: Target is version 19.0.0-1979-g6cec793d (squid) 2024-04-02T05:20:49.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:48 smithi192 ceph-mon[68000]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7bd5b84926d4a0d6c51e5562e965bff211cec2740c9f4dec9b5e00a2bbb75eae'] 2024-04-02T05:20:49.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:48 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:20:49.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:48 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:49.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:48 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all mgr 2024-04-02T05:20:49.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:48 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:49.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:48 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:49.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:48 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all mon 2024-04-02T05:20:49.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:48 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:49.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:48 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:49.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:48 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all crash 2024-04-02T05:20:49.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:48 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:49.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:48 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:49.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:48 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all osd 2024-04-02T05:20:49.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:48 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:49.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:48 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:49.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:48 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all mds 2024-04-02T05:20:49.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:48 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:49.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:48 smithi192 ceph-mon[68000]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 39 KiB/s rd, 0 B/s wr, 64 op/s 2024-04-02T05:20:49.698 INFO:teuthology.orchestra.run.smithi086.stdout:true 2024-04-02T05:20:50.358 INFO:teuthology.orchestra.run.smithi086.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T05:20:50.358 INFO:teuthology.orchestra.run.smithi086.stdout:alertmanager.a smithi086 *:9093,9094 running (8m) 2m ago 30m 26.1M - 0.25.0 c8568f914cd2 660ea62bb6b9 2024-04-02T05:20:50.358 INFO:teuthology.orchestra.run.smithi086.stdout:grafana.a smithi192 *:3000 running (8m) 29s ago 30m 79.0M - 9.4.12 2bacad6d85d8 259af9e63aa5 2024-04-02T05:20:50.359 INFO:teuthology.orchestra.run.smithi086.stdout:iscsi.foo.smithi086.rsbqwx smithi086 running (8m) 2m ago 29m 62.1M - 3.5 e1d6a67b021e fe728eb940f7 2024-04-02T05:20:50.359 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.x smithi192 *:8443,9283,8765 running (9m) 29s ago 35m 454M - 19.0.0-1979-g6cec793d 78f830190839 dd1e6d0f0a95 2024-04-02T05:20:50.359 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.y smithi086 *:8443,9283,8765 running (19m) 2m ago 36m 507M - 19.0.0-1979-g6cec793d 78f830190839 3dca54a018ab 2024-04-02T05:20:50.359 INFO:teuthology.orchestra.run.smithi086.stdout:mon.a smithi086 running (6m) 2m ago 36m 51.7M 2048M 19.0.0-1979-g6cec793d 78f830190839 8e478dd4da2d 2024-04-02T05:20:50.359 INFO:teuthology.orchestra.run.smithi086.stdout:mon.b smithi192 running (7m) 29s ago 35m 51.3M 2048M 19.0.0-1979-g6cec793d 78f830190839 4125e997188f 2024-04-02T05:20:50.359 INFO:teuthology.orchestra.run.smithi086.stdout:mon.c smithi086 running (6m) 2m ago 35m 42.9M 2048M 19.0.0-1979-g6cec793d 78f830190839 8180999b3142 2024-04-02T05:20:50.359 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.a smithi086 *:9100 running (9m) 2m ago 31m 16.3M - 1.5.0 0da6a335fe13 05a844569d7b 2024-04-02T05:20:50.359 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.b smithi192 *:9100 running (9m) 29s ago 31m 16.7M - 1.5.0 0da6a335fe13 1a7d6788d58b 2024-04-02T05:20:50.359 INFO:teuthology.orchestra.run.smithi086.stdout:osd.0 smithi086 running (5m) 2m ago 34m 46.0M 2992M 19.0.0-1979-g6cec793d 78f830190839 e521a0ef7daf 2024-04-02T05:20:50.359 INFO:teuthology.orchestra.run.smithi086.stdout:osd.1 smithi086 running (4m) 2m ago 34m 53.1M 2992M 19.0.0-1979-g6cec793d 78f830190839 76fe72844771 2024-04-02T05:20:50.359 INFO:teuthology.orchestra.run.smithi086.stdout:osd.2 smithi086 running (3m) 2m ago 34m 48.9M 2992M 19.0.0-1979-g6cec793d 78f830190839 28594f089a8f 2024-04-02T05:20:50.359 INFO:teuthology.orchestra.run.smithi086.stdout:osd.3 smithi086 running (2m) 2m ago 33m 12.3M 2992M 19.0.0-1979-g6cec793d 78f830190839 6028f927dfd6 2024-04-02T05:20:50.359 INFO:teuthology.orchestra.run.smithi086.stdout:osd.4 smithi192 running (2m) 29s ago 33m 52.4M 3248M 19.0.0-1979-g6cec793d 78f830190839 f9c58b761b62 2024-04-02T05:20:50.359 INFO:teuthology.orchestra.run.smithi086.stdout:osd.5 smithi192 running (104s) 29s ago 33m 51.3M 3248M 19.0.0-1979-g6cec793d 78f830190839 813afd5cf75d 2024-04-02T05:20:50.359 INFO:teuthology.orchestra.run.smithi086.stdout:osd.6 smithi192 running (69s) 29s ago 32m 51.6M 3248M 19.0.0-1979-g6cec793d 78f830190839 d3707b413a43 2024-04-02T05:20:50.360 INFO:teuthology.orchestra.run.smithi086.stdout:osd.7 smithi192 running (34s) 29s ago 32m 15.1M 3248M 19.0.0-1979-g6cec793d 78f830190839 48f6eb4f415f 2024-04-02T05:20:50.360 INFO:teuthology.orchestra.run.smithi086.stdout:prometheus.a smithi192 *:9095 running (9m) 29s ago 31m 72.6M - 2.43.0 a07b618ecd1d 8378fcdb60e3 2024-04-02T05:20:50.360 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi086.bxzxlp smithi086 *:8000 running (30m) 2m ago 30m 80.0M - 17.2.0 e1d6a67b021e 6725785fbc53 2024-04-02T05:20:50.360 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi192.pmlbot smithi192 *:8000 running (30m) 29s ago 30m 80.3M - 17.2.0 e1d6a67b021e 5ae3638f8b1c 2024-04-02T05:20:50.557 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:50 smithi086 ceph-mon[88839]: from='client.64558 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:20:50.557 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:50 smithi086 ceph-mon[88839]: from='client.54621 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:20:50.557 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:50 smithi086 ceph-mon[88839]: from='client.54627 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:20:50.557 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:50 smithi086 ceph-mon[88839]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 0 B/s wr, 97 op/s 2024-04-02T05:20:50.557 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:50 smithi086 ceph-mon[86443]: from='client.64558 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:20:50.557 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:50 smithi086 ceph-mon[86443]: from='client.54621 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:20:50.557 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:50 smithi086 ceph-mon[86443]: from='client.54627 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:20:50.557 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:50 smithi086 ceph-mon[86443]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 0 B/s wr, 97 op/s 2024-04-02T05:20:50.762 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:20:50.762 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": { 2024-04-02T05:20:50.762 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 3 2024-04-02T05:20:50.762 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:20:50.762 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": { 2024-04-02T05:20:50.762 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 2 2024-04-02T05:20:50.762 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:20:50.762 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": { 2024-04-02T05:20:50.762 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 8 2024-04-02T05:20:50.762 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:20:50.762 INFO:teuthology.orchestra.run.smithi086.stdout: "rgw": { 2024-04-02T05:20:50.763 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-02T05:20:50.763 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:20:50.763 INFO:teuthology.orchestra.run.smithi086.stdout: "overall": { 2024-04-02T05:20:50.763 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-04-02T05:20:50.763 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 13 2024-04-02T05:20:50.763 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-04-02T05:20:50.763 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:20:50.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:50 smithi192 ceph-mon[68000]: from='client.64558 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:20:50.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:50 smithi192 ceph-mon[68000]: from='client.54621 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:20:50.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:50 smithi192 ceph-mon[68000]: from='client.54627 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:20:50.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:50 smithi192 ceph-mon[68000]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 0 B/s wr, 97 op/s 2024-04-02T05:20:51.109 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:20:51.109 INFO:teuthology.orchestra.run.smithi086.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", 2024-04-02T05:20:51.109 INFO:teuthology.orchestra.run.smithi086.stdout: "in_progress": true, 2024-04-02T05:20:51.110 INFO:teuthology.orchestra.run.smithi086.stdout: "which": "Upgrading daemons in service(s) rgw.foo", 2024-04-02T05:20:51.110 INFO:teuthology.orchestra.run.smithi086.stdout: "services_complete": [], 2024-04-02T05:20:51.110 INFO:teuthology.orchestra.run.smithi086.stdout: "progress": "0/2 daemons upgraded", 2024-04-02T05:20:51.110 INFO:teuthology.orchestra.run.smithi086.stdout: "message": "Currently upgrading rgw daemons", 2024-04-02T05:20:51.110 INFO:teuthology.orchestra.run.smithi086.stdout: "is_paused": false 2024-04-02T05:20:51.110 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:20:51.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:51 smithi086 ceph-mon[86443]: Upgrade: Updating rgw.foo.smithi086.bxzxlp (1/2) 2024-04-02T05:20:51.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:51 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:51.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:51 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi086.bxzxlp", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T05:20:51.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:51 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi086.bxzxlp", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T05:20:51.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:51 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:20:51.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:51 smithi086 ceph-mon[86443]: Deploying daemon rgw.foo.smithi086.bxzxlp on smithi086 2024-04-02T05:20:51.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:51 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/137487283' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:51.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:51 smithi086 ceph-mon[86443]: from='client.54639 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:20:51.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:51 smithi086 ceph-mon[88839]: Upgrade: Updating rgw.foo.smithi086.bxzxlp (1/2) 2024-04-02T05:20:51.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:51 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:51.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:51 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi086.bxzxlp", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T05:20:51.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:51 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi086.bxzxlp", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T05:20:51.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:51 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:20:51.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:51 smithi086 ceph-mon[88839]: Deploying daemon rgw.foo.smithi086.bxzxlp on smithi086 2024-04-02T05:20:51.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:51 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/137487283' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:51.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:51 smithi086 ceph-mon[88839]: from='client.54639 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:20:51.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:51 smithi192 ceph-mon[68000]: Upgrade: Updating rgw.foo.smithi086.bxzxlp (1/2) 2024-04-02T05:20:51.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:51 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:51.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:51 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi086.bxzxlp", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T05:20:51.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:51 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi086.bxzxlp", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T05:20:51.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:51 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:20:51.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:51 smithi192 ceph-mon[68000]: Deploying daemon rgw.foo.smithi086.bxzxlp on smithi086 2024-04-02T05:20:51.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:51 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/137487283' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:20:51.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:51 smithi192 ceph-mon[68000]: from='client.54639 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:20:52.718 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:52 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:20:52.719 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:52 smithi086 ceph-mon[86443]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 69 KiB/s rd, 0 B/s wr, 113 op/s 2024-04-02T05:20:52.719 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:52 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:20:52.719 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:52 smithi086 ceph-mon[88839]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 69 KiB/s rd, 0 B/s wr, 113 op/s 2024-04-02T05:20:52.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:52 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:20:52.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:52 smithi192 ceph-mon[68000]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 69 KiB/s rd, 0 B/s wr, 113 op/s 2024-04-02T05:20:53.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:20:53 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:20:53] "GET /metrics HTTP/1.1" 200 38372 "" "Prometheus/2.43.0" 2024-04-02T05:20:54.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:54 smithi086 ceph-mon[86443]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-02T05:20:54.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:54 smithi086 ceph-mon[88839]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-02T05:20:54.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:54 smithi192 ceph-mon[68000]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-02T05:20:55.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:55 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:55.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:55 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:55.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:55 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:20:55.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:55 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:55.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:55 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:55.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:55 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:20:55.834 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:55 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:55.834 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:55 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:55.834 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:55 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:20:57.390 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:57 smithi192 ceph-mon[68000]: Upgrade: Updating rgw.foo.smithi192.pmlbot (2/2) 2024-04-02T05:20:57.390 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:57 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:57.390 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:57 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi192.pmlbot", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T05:20:57.390 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:57 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi192.pmlbot", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T05:20:57.390 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:57 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:20:57.390 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:57 smithi192 ceph-mon[68000]: Deploying daemon rgw.foo.smithi192.pmlbot on smithi192 2024-04-02T05:20:57.390 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:57 smithi192 ceph-mon[68000]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 98 KiB/s rd, 0 B/s wr, 160 op/s 2024-04-02T05:20:57.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:57 smithi086 ceph-mon[86443]: Upgrade: Updating rgw.foo.smithi192.pmlbot (2/2) 2024-04-02T05:20:57.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:57 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:57.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:57 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi192.pmlbot", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T05:20:57.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:57 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi192.pmlbot", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T05:20:57.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:57 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:20:57.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:57 smithi086 ceph-mon[86443]: Deploying daemon rgw.foo.smithi192.pmlbot on smithi192 2024-04-02T05:20:57.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:57 smithi086 ceph-mon[86443]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 98 KiB/s rd, 0 B/s wr, 160 op/s 2024-04-02T05:20:57.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:57 smithi086 ceph-mon[88839]: Upgrade: Updating rgw.foo.smithi192.pmlbot (2/2) 2024-04-02T05:20:57.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:57 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:20:57.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:57 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi192.pmlbot", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T05:20:57.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:57 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi192.pmlbot", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T05:20:57.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:57 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:20:57.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:57 smithi086 ceph-mon[88839]: Deploying daemon rgw.foo.smithi192.pmlbot on smithi192 2024-04-02T05:20:57.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:57 smithi086 ceph-mon[88839]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 98 KiB/s rd, 0 B/s wr, 160 op/s 2024-04-02T05:20:58.654 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:20:58 smithi192 ceph-mon[68000]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 128 KiB/s rd, 85 B/s wr, 207 op/s 2024-04-02T05:20:58.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:20:58 smithi086 ceph-mon[86443]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 128 KiB/s rd, 85 B/s wr, 207 op/s 2024-04-02T05:20:58.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:20:58 smithi086 ceph-mon[88839]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 128 KiB/s rd, 85 B/s wr, 207 op/s 2024-04-02T05:21:01.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:01.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:01 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:01.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:01 smithi086 ceph-mon[86443]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 123 KiB/s rd, 85 B/s wr, 196 op/s 2024-04-02T05:21:01.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:01.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:01 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:01.334 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:01 smithi086 ceph-mon[88839]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 123 KiB/s rd, 85 B/s wr, 196 op/s 2024-04-02T05:21:01.426 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:01.426 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:01 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:01.426 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:01 smithi192 ceph-mon[68000]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 123 KiB/s rd, 85 B/s wr, 196 op/s 2024-04-02T05:21:02.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:02 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:21:02.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:02 smithi192 ceph-mon[68000]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 123 KiB/s rd, 85 B/s wr, 196 op/s 2024-04-02T05:21:02.740 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:02 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:21:02.740 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:02 smithi086 ceph-mon[88839]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 123 KiB/s rd, 85 B/s wr, 196 op/s 2024-04-02T05:21:02.740 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:02 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:21:02.740 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:02 smithi086 ceph-mon[86443]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 123 KiB/s rd, 85 B/s wr, 196 op/s 2024-04-02T05:21:03.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:21:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:21:03] "GET /metrics HTTP/1.1" 200 38371 "" "Prometheus/2.43.0" 2024-04-02T05:21:05.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:04 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:05.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:04 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:05.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:04 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:05.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:04 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:05.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:04 smithi086 ceph-mon[86443]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 148 KiB/s rd, 170 B/s wr, 233 op/s 2024-04-02T05:21:05.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:04 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:05.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:04 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:05.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:04 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:05.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:04 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:05.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:04 smithi086 ceph-mon[88839]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 148 KiB/s rd, 170 B/s wr, 233 op/s 2024-04-02T05:21:05.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:04 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:05.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:04 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:05.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:04 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:05.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:04 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:05.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:04 smithi192 ceph-mon[68000]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 148 KiB/s rd, 170 B/s wr, 233 op/s 2024-04-02T05:21:06.048 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:06 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:06.048 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:06 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:06.048 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:06 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:06.048 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:06 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:06.048 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:06 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:06.048 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:06 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:06.049 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:06 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:06.049 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:06 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:06.392 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:06 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:06.392 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:06 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:06.392 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:06 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:06.392 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:06 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:07.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:07 smithi086 ceph-mon[86443]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 157 KiB/s rd, 170 B/s wr, 248 op/s 2024-04-02T05:21:07.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:07 smithi086 ceph-mon[88839]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 157 KiB/s rd, 170 B/s wr, 248 op/s 2024-04-02T05:21:07.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:07 smithi192 ceph-mon[68000]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 157 KiB/s rd, 170 B/s wr, 248 op/s 2024-04-02T05:21:08.541 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:08 smithi192 ceph-mon[68000]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 132 KiB/s rd, 170 B/s wr, 208 op/s 2024-04-02T05:21:08.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:08 smithi086 ceph-mon[88839]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 132 KiB/s rd, 170 B/s wr, 208 op/s 2024-04-02T05:21:08.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:08 smithi086 ceph-mon[86443]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 132 KiB/s rd, 170 B/s wr, 208 op/s 2024-04-02T05:21:09.928 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:09.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:09.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:09.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:09.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:21:09.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:21:09.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:09.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:21:09.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:09.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:09.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:09.929 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:09.930 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:09.930 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:09.930 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all rgw 2024-04-02T05:21:09.930 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:09.930 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi086.bxzxlp"}]: dispatch 2024-04-02T05:21:09.930 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi086.bxzxlp"}]: dispatch 2024-04-02T05:21:09.930 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi086.bxzxlp"}]': finished 2024-04-02T05:21:09.930 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi192.pmlbot"}]: dispatch 2024-04-02T05:21:09.930 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi192.pmlbot"}]: dispatch 2024-04-02T05:21:09.930 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi192.pmlbot"}]': finished 2024-04-02T05:21:09.931 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:09.931 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all rbd-mirror 2024-04-02T05:21:09.931 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:09.931 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:09.931 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:09.931 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all ceph-exporter 2024-04-02T05:21:09.931 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:09.931 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:09.931 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all nfs 2024-04-02T05:21:09.931 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:09.931 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:09.931 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all nvmeof 2024-04-02T05:21:09.932 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:09.932 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:09.932 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:09.932 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:09.932 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:09.932 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:09.932 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:09.932 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: Upgrade: Finalizing container_image settings 2024-04-02T05:21:09.932 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:21:09.932 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:21:09.932 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-02T05:21:09.932 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:09.933 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:09.933 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-02T05:21:09.933 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:21:09.933 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:21:09.933 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-02T05:21:09.933 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:21:09.933 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:21:09.933 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-02T05:21:09.933 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:21:09.933 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:21:09.933 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-02T05:21:09.933 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:21:09.934 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:21:09.934 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-04-02T05:21:09.934 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:21:09.934 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:21:09.934 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-02T05:21:09.934 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:09.934 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:09.934 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:21:09.934 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:21:09.934 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-02T05:21:09.934 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:21:09.935 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:21:09.935 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:21:09.935 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:21:09.935 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-02T05:21:09.935 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:21:09.935 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:21:09.935 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-02T05:21:09.935 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:09.935 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:09.935 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:09.935 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:09.935 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:09.936 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:09.936 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:09.936 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:09.936 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:09.936 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:09.936 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:09.936 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:09.936 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: Upgrade: Complete! 2024-04-02T05:21:09.936 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:21:09.936 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:21:09.936 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-02T05:21:09.936 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:21:09.937 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:21:09.937 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: Checking dashboard <-> RGW credentials 2024-04-02T05:21:09.937 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:09.937 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:21:09.937 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:21:09.937 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:09 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:10.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:10.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:10.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:10.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:10.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:21:10.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:21:10.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:10.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:21:10.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:10.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:10.084 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:10.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:10.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:10.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:10.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all rgw 2024-04-02T05:21:10.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:10.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi086.bxzxlp"}]: dispatch 2024-04-02T05:21:10.085 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi086.bxzxlp"}]: dispatch 2024-04-02T05:21:10.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi086.bxzxlp"}]': finished 2024-04-02T05:21:10.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi192.pmlbot"}]: dispatch 2024-04-02T05:21:10.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi192.pmlbot"}]: dispatch 2024-04-02T05:21:10.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi192.pmlbot"}]': finished 2024-04-02T05:21:10.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:10.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all rbd-mirror 2024-04-02T05:21:10.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:10.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:10.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:10.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all ceph-exporter 2024-04-02T05:21:10.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:10.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:10.086 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all nfs 2024-04-02T05:21:10.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:10.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:10.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:10.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:10.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:21:10.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:21:10.087 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:10.088 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:21:10.088 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:10.088 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:10.088 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:10.088 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:10.088 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:10.088 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:10.088 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all rgw 2024-04-02T05:21:10.088 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:10.088 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi086.bxzxlp"}]: dispatch 2024-04-02T05:21:10.088 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi086.bxzxlp"}]: dispatch 2024-04-02T05:21:10.088 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi086.bxzxlp"}]': finished 2024-04-02T05:21:10.088 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi192.pmlbot"}]: dispatch 2024-04-02T05:21:10.088 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi192.pmlbot"}]: dispatch 2024-04-02T05:21:10.088 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi192.pmlbot"}]': finished 2024-04-02T05:21:10.088 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:10.089 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all rbd-mirror 2024-04-02T05:21:10.089 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:10.089 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:10.089 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:10.089 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all ceph-exporter 2024-04-02T05:21:10.089 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:10.089 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:10.089 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all nfs 2024-04-02T05:21:10.089 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:10.089 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:10.089 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all nvmeof 2024-04-02T05:21:10.089 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:10.089 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:10.089 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:10.089 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:10.090 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:10.090 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:10.090 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:10.090 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: Upgrade: Finalizing container_image settings 2024-04-02T05:21:10.090 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:21:10.090 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:21:10.090 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-02T05:21:10.090 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:10.090 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:10.090 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-02T05:21:10.090 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:21:10.090 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:21:10.090 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-02T05:21:10.090 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:21:10.091 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:21:10.091 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-02T05:21:10.091 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:21:10.091 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:21:10.091 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-02T05:21:10.091 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:21:10.092 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:10.092 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:10.092 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all nvmeof 2024-04-02T05:21:10.092 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:10.092 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:10.092 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:10.092 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:10.092 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:10.092 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:10.092 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:10.092 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: Upgrade: Finalizing container_image settings 2024-04-02T05:21:10.092 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:21:10.092 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-02T05:21:10.092 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-02T05:21:10.092 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:10.093 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:10.093 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-02T05:21:10.093 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:21:10.093 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-02T05:21:10.093 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-02T05:21:10.093 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:21:10.093 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-02T05:21:10.093 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-02T05:21:10.093 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:21:10.093 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-02T05:21:10.093 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-02T05:21:10.093 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:21:10.094 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:21:10.094 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-04-02T05:21:10.094 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:21:10.094 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:21:10.094 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-02T05:21:10.094 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:10.094 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:10.094 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:21:10.094 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:21:10.094 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-02T05:21:10.094 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:21:10.094 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:21:10.095 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:21:10.095 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:21:10.095 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-02T05:21:10.095 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:21:10.095 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:21:10.095 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-02T05:21:10.095 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:10.095 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:10.095 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:10.095 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:10.095 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:10.095 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:10.095 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-02T05:21:10.096 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-04-02T05:21:10.096 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:21:10.096 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-02T05:21:10.096 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-02T05:21:10.096 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:10.096 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:10.096 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:21:10.096 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-02T05:21:10.096 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-02T05:21:10.096 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:21:10.096 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-02T05:21:10.096 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:21:10.096 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-02T05:21:10.096 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-02T05:21:10.096 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:21:10.097 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-02T05:21:10.097 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-02T05:21:10.097 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:10.097 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:10.097 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:10.097 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:10.097 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:10.097 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:10.097 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:10.097 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:10.097 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:10.098 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:10.098 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:10.098 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:10.098 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: Upgrade: Complete! 2024-04-02T05:21:10.098 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:21:10.098 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:21:10.098 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-02T05:21:10.098 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:21:10.098 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:21:10.098 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: Checking dashboard <-> RGW credentials 2024-04-02T05:21:10.098 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:10.098 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:21:10.098 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:21:10.099 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:10.099 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:10.099 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:10.099 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:10.099 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:10.099 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:10.099 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-02T05:21:10.099 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: Upgrade: Complete! 2024-04-02T05:21:10.099 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:21:10.099 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-02T05:21:10.099 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-02T05:21:10.099 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:21:10.099 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:21:10.100 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: Checking dashboard <-> RGW credentials 2024-04-02T05:21:10.100 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:10.100 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:21:10.100 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:21:10.100 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:09 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:10.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:10 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:21:10.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:10 smithi192 ceph-mon[68000]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 90 KiB/s rd, 85 B/s wr, 141 op/s 2024-04-02T05:21:11.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:10 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:21:11.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:10 smithi086 ceph-mon[86443]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 90 KiB/s rd, 85 B/s wr, 141 op/s 2024-04-02T05:21:11.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:10 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:21:11.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:10 smithi086 ceph-mon[88839]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 90 KiB/s rd, 85 B/s wr, 141 op/s 2024-04-02T05:21:11.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:11 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:12.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:11 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:12.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:11 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:12.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:12 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:21:12.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:12 smithi192 ceph-mon[68000]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 69 KiB/s rd, 85 B/s wr, 109 op/s 2024-04-02T05:21:13.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:12 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:21:13.083 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:12 smithi086 ceph-mon[88839]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 69 KiB/s rd, 85 B/s wr, 109 op/s 2024-04-02T05:21:13.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:12 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:21:13.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:12 smithi086 ceph-mon[86443]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 69 KiB/s rd, 85 B/s wr, 109 op/s 2024-04-02T05:21:13.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:21:13 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:21:13] "GET /metrics HTTP/1.1" 200 38406 "" "Prometheus/2.43.0" 2024-04-02T05:21:14.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:14 smithi086 ceph-mon[86443]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 50 KiB/s rd, 85 B/s wr, 77 op/s 2024-04-02T05:21:14.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:14 smithi086 ceph-mon[88839]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 50 KiB/s rd, 85 B/s wr, 77 op/s 2024-04-02T05:21:14.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:14 smithi192 ceph-mon[68000]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 50 KiB/s rd, 85 B/s wr, 77 op/s 2024-04-02T05:21:16.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:16 smithi086 ceph-mon[86443]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 23 op/s 2024-04-02T05:21:16.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:16 smithi086 ceph-mon[88839]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 23 op/s 2024-04-02T05:21:16.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:16 smithi192 ceph-mon[68000]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 23 op/s 2024-04-02T05:21:18.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:18 smithi086 ceph-mon[88839]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-04-02T05:21:18.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:18 smithi086 ceph-mon[86443]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-04-02T05:21:18.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:18 smithi192 ceph-mon[68000]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-04-02T05:21:20.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:20 smithi086 ceph-mon[86443]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-04-02T05:21:20.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:20 smithi086 ceph-mon[88839]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-04-02T05:21:20.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:20 smithi192 ceph-mon[68000]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s 2024-04-02T05:21:22.239 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch ps' 2024-04-02T05:21:22.551 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:22 smithi086 ceph-mon[88839]: from='client.54723 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:21:22.552 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:22 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:21:22.552 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:22 smithi086 ceph-mon[88839]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 2 op/s 2024-04-02T05:21:22.552 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:22 smithi086 ceph-mon[86443]: from='client.54723 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:21:22.552 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:22 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:21:22.552 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:22 smithi086 ceph-mon[86443]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 2 op/s 2024-04-02T05:21:22.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:22 smithi192 ceph-mon[68000]: from='client.54723 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:21:22.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:22 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:21:22.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:22 smithi192 ceph-mon[68000]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 2 op/s 2024-04-02T05:21:23.495 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:21:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:21:23] "GET /metrics HTTP/1.1" 200 38406 "" "Prometheus/2.43.0" 2024-04-02T05:21:24.149 INFO:teuthology.orchestra.run.smithi086.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T05:21:24.149 INFO:teuthology.orchestra.run.smithi086.stdout:alertmanager.a smithi086 *:9093,9094 running (9m) 20s ago 31m 25.7M - 0.25.0 c8568f914cd2 660ea62bb6b9 2024-04-02T05:21:24.149 INFO:teuthology.orchestra.run.smithi086.stdout:grafana.a smithi192 *:3000 running (9m) 20s ago 31m 75.5M - 9.4.12 2bacad6d85d8 259af9e63aa5 2024-04-02T05:21:24.149 INFO:teuthology.orchestra.run.smithi086.stdout:iscsi.foo.smithi086.rsbqwx smithi086 running (8m) 20s ago 30m 64.5M - 3.5 e1d6a67b021e fe728eb940f7 2024-04-02T05:21:24.149 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.x smithi192 *:8443,9283,8765 running (10m) 20s ago 35m 454M - 19.0.0-1979-g6cec793d 78f830190839 dd1e6d0f0a95 2024-04-02T05:21:24.149 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.y smithi086 *:8443,9283,8765 running (20m) 20s ago 37m 512M - 19.0.0-1979-g6cec793d 78f830190839 3dca54a018ab 2024-04-02T05:21:24.150 INFO:teuthology.orchestra.run.smithi086.stdout:mon.a smithi086 running (7m) 20s ago 37m 59.0M 2048M 19.0.0-1979-g6cec793d 78f830190839 8e478dd4da2d 2024-04-02T05:21:24.150 INFO:teuthology.orchestra.run.smithi086.stdout:mon.b smithi192 running (8m) 20s ago 35m 52.5M 2048M 19.0.0-1979-g6cec793d 78f830190839 4125e997188f 2024-04-02T05:21:24.150 INFO:teuthology.orchestra.run.smithi086.stdout:mon.c smithi086 running (6m) 20s ago 36m 48.7M 2048M 19.0.0-1979-g6cec793d 78f830190839 8180999b3142 2024-04-02T05:21:24.150 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.a smithi086 *:9100 running (10m) 20s ago 32m 15.3M - 1.5.0 0da6a335fe13 05a844569d7b 2024-04-02T05:21:24.150 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.b smithi192 *:9100 running (10m) 20s ago 32m 16.5M - 1.5.0 0da6a335fe13 1a7d6788d58b 2024-04-02T05:21:24.150 INFO:teuthology.orchestra.run.smithi086.stdout:osd.0 smithi086 running (5m) 20s ago 35m 55.4M 2992M 19.0.0-1979-g6cec793d 78f830190839 e521a0ef7daf 2024-04-02T05:21:24.150 INFO:teuthology.orchestra.run.smithi086.stdout:osd.1 smithi086 running (5m) 20s ago 35m 65.1M 2992M 19.0.0-1979-g6cec793d 78f830190839 76fe72844771 2024-04-02T05:21:24.150 INFO:teuthology.orchestra.run.smithi086.stdout:osd.2 smithi086 running (4m) 20s ago 34m 56.8M 2992M 19.0.0-1979-g6cec793d 78f830190839 28594f089a8f 2024-04-02T05:21:24.150 INFO:teuthology.orchestra.run.smithi086.stdout:osd.3 smithi086 running (3m) 20s ago 34m 51.4M 2992M 19.0.0-1979-g6cec793d 78f830190839 6028f927dfd6 2024-04-02T05:21:24.150 INFO:teuthology.orchestra.run.smithi086.stdout:osd.4 smithi192 running (2m) 20s ago 33m 58.2M 3248M 19.0.0-1979-g6cec793d 78f830190839 f9c58b761b62 2024-04-02T05:21:24.150 INFO:teuthology.orchestra.run.smithi086.stdout:osd.5 smithi192 running (2m) 20s ago 33m 56.7M 3248M 19.0.0-1979-g6cec793d 78f830190839 813afd5cf75d 2024-04-02T05:21:24.150 INFO:teuthology.orchestra.run.smithi086.stdout:osd.6 smithi192 running (103s) 20s ago 33m 55.9M 3248M 19.0.0-1979-g6cec793d 78f830190839 d3707b413a43 2024-04-02T05:21:24.150 INFO:teuthology.orchestra.run.smithi086.stdout:osd.7 smithi192 running (68s) 20s ago 32m 56.3M 3248M 19.0.0-1979-g6cec793d 78f830190839 48f6eb4f415f 2024-04-02T05:21:24.150 INFO:teuthology.orchestra.run.smithi086.stdout:prometheus.a smithi192 *:9095 running (9m) 20s ago 32m 72.6M - 2.43.0 a07b618ecd1d 8378fcdb60e3 2024-04-02T05:21:24.150 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi086.bxzxlp smithi086 *:8000 running (30s) 20s ago 30m 76.2M - 19.0.0-1979-g6cec793d 78f830190839 8c2d4c7203c3 2024-04-02T05:21:24.151 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi192.pmlbot smithi192 *:8000 running (24s) 20s ago 30m 75.2M - 19.0.0-1979-g6cec793d 78f830190839 5adcaee89616 2024-04-02T05:21:24.744 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:24 smithi086 ceph-mon[86443]: from='client.54729 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:21:24.744 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:24 smithi086 ceph-mon[86443]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 2 op/s 2024-04-02T05:21:24.744 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:24 smithi086 ceph-mon[88839]: from='client.54729 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:21:24.744 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:24 smithi086 ceph-mon[88839]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 2 op/s 2024-04-02T05:21:24.768 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph versions | jq -e '"'"'.rgw | length == 1'"'"'' 2024-04-02T05:21:24.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:24 smithi192 ceph-mon[68000]: from='client.54729 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:21:24.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:24 smithi192 ceph-mon[68000]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 2 op/s 2024-04-02T05:21:25.650 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:25 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:21:25.650 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:25 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:21:25.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:25 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:21:26.705 INFO:teuthology.orchestra.run.smithi086.stdout:true 2024-04-02T05:21:26.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:26 smithi086 ceph-mon[86443]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:21:26.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:26 smithi086 ceph-mon[88839]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:21:26.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:26 smithi192 ceph-mon[68000]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:21:27.365 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph versions | jq -e '"'"'.rgw | keys'"'"' | grep $sha1' 2024-04-02T05:21:27.673 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:27 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/300049341' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:27.674 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:27 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/300049341' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:27.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:27 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/300049341' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:28.671 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:28 smithi086 ceph-mon[88839]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:21:28.672 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:28 smithi086 ceph-mon[86443]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:21:28.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:28 smithi192 ceph-mon[68000]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:21:29.458 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)" 2024-04-02T05:21:29.745 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:29 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/2188047223' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:29.745 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:29 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/2188047223' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:29.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:29 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/2188047223' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:30.044 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch upgrade status' 2024-04-02T05:21:30.601 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:30 smithi086 ceph-mon[88839]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:21:30.601 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:30 smithi086 ceph-mon[86443]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:21:30.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:30 smithi192 ceph-mon[68000]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:21:32.153 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:21:32.153 INFO:teuthology.orchestra.run.smithi086.stdout: "target_image": null, 2024-04-02T05:21:32.154 INFO:teuthology.orchestra.run.smithi086.stdout: "in_progress": false, 2024-04-02T05:21:32.154 INFO:teuthology.orchestra.run.smithi086.stdout: "which": "", 2024-04-02T05:21:32.154 INFO:teuthology.orchestra.run.smithi086.stdout: "services_complete": [], 2024-04-02T05:21:32.154 INFO:teuthology.orchestra.run.smithi086.stdout: "progress": null, 2024-04-02T05:21:32.154 INFO:teuthology.orchestra.run.smithi086.stdout: "message": "", 2024-04-02T05:21:32.154 INFO:teuthology.orchestra.run.smithi086.stdout: "is_paused": false 2024-04-02T05:21:32.154 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:21:32.825 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:32 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:21:32.825 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:32 smithi086 ceph-mon[88839]: from='client.54747 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:21:32.825 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:32 smithi086 ceph-mon[88839]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:21:32.825 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:32 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:21:32.825 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:32 smithi086 ceph-mon[86443]: from='client.54747 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:21:32.826 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:32 smithi086 ceph-mon[86443]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:21:32.848 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph health detail' 2024-04-02T05:21:32.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:32 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:21:32.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:32 smithi192 ceph-mon[68000]: from='client.54747 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:21:32.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:32 smithi192 ceph-mon[68000]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:21:33.478 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:21:33 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:21:33] "GET /metrics HTTP/1.1" 200 38405 "" "Prometheus/2.43.0" 2024-04-02T05:21:34.507 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:34 smithi086 ceph-mon[88839]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:21:34.507 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:34 smithi086 ceph-mon[86443]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:21:34.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:34 smithi192 ceph-mon[68000]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:21:35.007 INFO:teuthology.orchestra.run.smithi086.stdout:HEALTH_OK 2024-04-02T05:21:35.582 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:35 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/778770525' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T05:21:35.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:35 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/778770525' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T05:21:35.746 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-02T05:21:35.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:35 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/778770525' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T05:21:36.828 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:36 smithi192 ceph-mon[68000]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:21:36.832 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:36 smithi086 ceph-mon[86443]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:21:36.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:36 smithi086 ceph-mon[88839]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:21:37.685 INFO:teuthology.orchestra.run.smithi086.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:21:38.346 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-02T05:21:38.355 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi086.front.sepia.ceph.com 2024-04-02T05:21:38.356 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-04-02T05:21:38.929 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:38 smithi086 ceph-mon[88839]: from='client.64702 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:21:38.929 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:38 smithi086 ceph-mon[88839]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:21:38.930 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:38 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:38.930 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:38 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:21:38.930 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:38 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:21:38.930 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:38 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:38.930 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:38 smithi086 ceph-mon[88839]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:21:38.930 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:38 smithi086 ceph-mon[88839]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:21:38.930 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:38 smithi086 ceph-mon[86443]: from='client.64702 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:21:38.930 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:38 smithi086 ceph-mon[86443]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:21:38.930 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:38 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:38.930 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:38 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:21:38.930 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:38 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:21:38.931 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:38 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:38.931 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:38 smithi086 ceph-mon[86443]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:21:38.931 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:38 smithi086 ceph-mon[86443]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:21:39.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:38 smithi192 ceph-mon[68000]: from='client.64702 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:21:39.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:38 smithi192 ceph-mon[68000]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:21:39.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:38 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:39.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:38 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:21:39.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:38 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:21:39.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:38 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:39.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:38 smithi192 ceph-mon[68000]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:21:39.178 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:38 smithi192 ceph-mon[68000]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:21:41.535 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:41.535 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[88839]: Upgrade: Target is version 19.0.0-1979-g6cec793d (squid) 2024-04-02T05:21:41.535 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[88839]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7bd5b84926d4a0d6c51e5562e965bff211cec2740c9f4dec9b5e00a2bbb75eae'] 2024-04-02T05:21:41.535 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:21:41.535 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:41.536 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all mgr 2024-04-02T05:21:41.536 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:41.536 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:41.536 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all mon 2024-04-02T05:21:41.536 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:41.536 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:41.536 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all crash 2024-04-02T05:21:41.536 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:41.536 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:41.536 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all osd 2024-04-02T05:21:41.536 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:41.536 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:41.536 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all mds 2024-04-02T05:21:41.536 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:41.537 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:41.537 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all rgw 2024-04-02T05:21:41.537 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:41.537 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:41.537 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all rbd-mirror 2024-04-02T05:21:41.537 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:41.537 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:41.537 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-02T05:21:41.537 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:41.537 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[88839]: Upgrade: Setting container_image for all ceph-exporter 2024-04-02T05:21:41.537 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:41.537 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:21:41.537 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[88839]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:21:41.538 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:41.538 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[86443]: Upgrade: Target is version 19.0.0-1979-g6cec793d (squid) 2024-04-02T05:21:41.538 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[86443]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7bd5b84926d4a0d6c51e5562e965bff211cec2740c9f4dec9b5e00a2bbb75eae'] 2024-04-02T05:21:41.538 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:21:41.538 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:41.538 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all mgr 2024-04-02T05:21:41.538 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:41.538 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:41.538 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all mon 2024-04-02T05:21:41.538 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:41.538 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:41.538 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all crash 2024-04-02T05:21:41.539 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:41.539 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:41.539 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all osd 2024-04-02T05:21:41.539 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:41.539 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:41.539 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all mds 2024-04-02T05:21:41.539 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:41.539 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:41.539 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all rgw 2024-04-02T05:21:41.539 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:41.539 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:41.539 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all rbd-mirror 2024-04-02T05:21:41.539 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:41.539 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:41.540 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-02T05:21:41.540 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:41.540 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[86443]: Upgrade: Setting container_image for all ceph-exporter 2024-04-02T05:21:41.540 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:41.540 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:21:41.540 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:41 smithi086 ceph-mon[86443]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:21:41.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:41 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:41.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:41 smithi192 ceph-mon[68000]: Upgrade: Target is version 19.0.0-1979-g6cec793d (squid) 2024-04-02T05:21:41.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:41 smithi192 ceph-mon[68000]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7bd5b84926d4a0d6c51e5562e965bff211cec2740c9f4dec9b5e00a2bbb75eae'] 2024-04-02T05:21:41.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:41 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T05:21:41.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:41 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:41.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:41 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all mgr 2024-04-02T05:21:41.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:41 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:41.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:41 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:41.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:41 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all mon 2024-04-02T05:21:41.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:41 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:41.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:41 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:41.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:41 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all crash 2024-04-02T05:21:41.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:41 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:41.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:41 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:41.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:41 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all osd 2024-04-02T05:21:41.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:41 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:41.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:41 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:41.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:41 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all mds 2024-04-02T05:21:41.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:41 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:41.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:41 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:41.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:41 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all rgw 2024-04-02T05:21:41.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:41 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:41.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:41 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:41.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:41 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all rbd-mirror 2024-04-02T05:21:41.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:41 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:41.679 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:41 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:41.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:41 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-02T05:21:41.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:41 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:41.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:41 smithi192 ceph-mon[68000]: Upgrade: Setting container_image for all ceph-exporter 2024-04-02T05:21:41.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:41 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:41.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:41 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:21:41.680 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:41 smithi192 ceph-mon[68000]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:21:41.775 INFO:teuthology.orchestra.run.smithi086.stdout:true 2024-04-02T05:21:42.438 INFO:teuthology.orchestra.run.smithi086.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T05:21:42.438 INFO:teuthology.orchestra.run.smithi086.stdout:alertmanager.a smithi086 *:9093,9094 running (9m) 38s ago 31m 25.7M - 0.25.0 c8568f914cd2 660ea62bb6b9 2024-04-02T05:21:42.438 INFO:teuthology.orchestra.run.smithi086.stdout:grafana.a smithi192 *:3000 running (9m) 38s ago 31m 75.5M - 9.4.12 2bacad6d85d8 259af9e63aa5 2024-04-02T05:21:42.438 INFO:teuthology.orchestra.run.smithi086.stdout:iscsi.foo.smithi086.rsbqwx smithi086 running (9m) 38s ago 30m 64.5M - 3.5 e1d6a67b021e fe728eb940f7 2024-04-02T05:21:42.438 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.x smithi192 *:8443,9283,8765 running (10m) 38s ago 36m 454M - 19.0.0-1979-g6cec793d 78f830190839 dd1e6d0f0a95 2024-04-02T05:21:42.438 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.y smithi086 *:8443,9283,8765 running (20m) 38s ago 37m 512M - 19.0.0-1979-g6cec793d 78f830190839 3dca54a018ab 2024-04-02T05:21:42.438 INFO:teuthology.orchestra.run.smithi086.stdout:mon.a smithi086 running (7m) 38s ago 37m 59.0M 2048M 19.0.0-1979-g6cec793d 78f830190839 8e478dd4da2d 2024-04-02T05:21:42.438 INFO:teuthology.orchestra.run.smithi086.stdout:mon.b smithi192 running (8m) 38s ago 36m 52.5M 2048M 19.0.0-1979-g6cec793d 78f830190839 4125e997188f 2024-04-02T05:21:42.438 INFO:teuthology.orchestra.run.smithi086.stdout:mon.c smithi086 running (7m) 38s ago 36m 48.7M 2048M 19.0.0-1979-g6cec793d 78f830190839 8180999b3142 2024-04-02T05:21:42.438 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.a smithi086 *:9100 running (10m) 38s ago 32m 15.3M - 1.5.0 0da6a335fe13 05a844569d7b 2024-04-02T05:21:42.439 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.b smithi192 *:9100 running (10m) 38s ago 32m 16.5M - 1.5.0 0da6a335fe13 1a7d6788d58b 2024-04-02T05:21:42.439 INFO:teuthology.orchestra.run.smithi086.stdout:osd.0 smithi086 running (6m) 38s ago 35m 55.4M 2992M 19.0.0-1979-g6cec793d 78f830190839 e521a0ef7daf 2024-04-02T05:21:42.439 INFO:teuthology.orchestra.run.smithi086.stdout:osd.1 smithi086 running (5m) 38s ago 35m 65.1M 2992M 19.0.0-1979-g6cec793d 78f830190839 76fe72844771 2024-04-02T05:21:42.439 INFO:teuthology.orchestra.run.smithi086.stdout:osd.2 smithi086 running (4m) 38s ago 35m 56.8M 2992M 19.0.0-1979-g6cec793d 78f830190839 28594f089a8f 2024-04-02T05:21:42.439 INFO:teuthology.orchestra.run.smithi086.stdout:osd.3 smithi086 running (3m) 38s ago 34m 51.4M 2992M 19.0.0-1979-g6cec793d 78f830190839 6028f927dfd6 2024-04-02T05:21:42.439 INFO:teuthology.orchestra.run.smithi086.stdout:osd.4 smithi192 running (3m) 38s ago 34m 58.2M 3248M 19.0.0-1979-g6cec793d 78f830190839 f9c58b761b62 2024-04-02T05:21:42.439 INFO:teuthology.orchestra.run.smithi086.stdout:osd.5 smithi192 running (2m) 38s ago 33m 56.7M 3248M 19.0.0-1979-g6cec793d 78f830190839 813afd5cf75d 2024-04-02T05:21:42.439 INFO:teuthology.orchestra.run.smithi086.stdout:osd.6 smithi192 running (2m) 38s ago 33m 55.9M 3248M 19.0.0-1979-g6cec793d 78f830190839 d3707b413a43 2024-04-02T05:21:42.439 INFO:teuthology.orchestra.run.smithi086.stdout:osd.7 smithi192 running (86s) 38s ago 33m 56.3M 3248M 19.0.0-1979-g6cec793d 78f830190839 48f6eb4f415f 2024-04-02T05:21:42.439 INFO:teuthology.orchestra.run.smithi086.stdout:prometheus.a smithi192 *:9095 running (10m) 38s ago 32m 72.6M - 2.43.0 a07b618ecd1d 8378fcdb60e3 2024-04-02T05:21:42.439 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi086.bxzxlp smithi086 *:8000 running (48s) 38s ago 30m 76.2M - 19.0.0-1979-g6cec793d 78f830190839 8c2d4c7203c3 2024-04-02T05:21:42.439 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi192.pmlbot smithi192 *:8000 running (42s) 38s ago 30m 75.2M - 19.0.0-1979-g6cec793d 78f830190839 5adcaee89616 2024-04-02T05:21:42.849 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:21:42.849 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": { 2024-04-02T05:21:42.849 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 3 2024-04-02T05:21:42.849 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:21:42.849 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": { 2024-04-02T05:21:42.849 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 2 2024-04-02T05:21:42.850 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:21:42.850 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": { 2024-04-02T05:21:42.850 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 8 2024-04-02T05:21:42.850 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:21:42.850 INFO:teuthology.orchestra.run.smithi086.stdout: "rgw": { 2024-04-02T05:21:42.850 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 2 2024-04-02T05:21:42.850 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:21:42.850 INFO:teuthology.orchestra.run.smithi086.stdout: "overall": { 2024-04-02T05:21:42.850 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 15 2024-04-02T05:21:42.851 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-04-02T05:21:42.851 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:21:43.195 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:21:43.195 INFO:teuthology.orchestra.run.smithi086.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", 2024-04-02T05:21:43.195 INFO:teuthology.orchestra.run.smithi086.stdout: "in_progress": true, 2024-04-02T05:21:43.196 INFO:teuthology.orchestra.run.smithi086.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-02T05:21:43.196 INFO:teuthology.orchestra.run.smithi086.stdout: "services_complete": [ 2024-04-02T05:21:43.196 INFO:teuthology.orchestra.run.smithi086.stdout: "rgw", 2024-04-02T05:21:43.196 INFO:teuthology.orchestra.run.smithi086.stdout: "mon", 2024-04-02T05:21:43.196 INFO:teuthology.orchestra.run.smithi086.stdout: "osd", 2024-04-02T05:21:43.196 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr" 2024-04-02T05:21:43.196 INFO:teuthology.orchestra.run.smithi086.stdout: ], 2024-04-02T05:21:43.196 INFO:teuthology.orchestra.run.smithi086.stdout: "progress": "15/21 daemons upgraded", 2024-04-02T05:21:43.196 INFO:teuthology.orchestra.run.smithi086.stdout: "message": "Currently upgrading iscsi daemons", 2024-04-02T05:21:43.196 INFO:teuthology.orchestra.run.smithi086.stdout: "is_paused": false 2024-04-02T05:21:43.196 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:21:43.590 INFO:teuthology.orchestra.run.smithi086.stdout:HEALTH_OK 2024-04-02T05:21:43.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:43 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:21:43.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:43 smithi192 ceph-mon[68000]: from='client.54765 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:21:43.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:43 smithi192 ceph-mon[68000]: from='client.64711 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:21:43.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:43 smithi192 ceph-mon[68000]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:21:43.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:43 smithi192 ceph-mon[68000]: Upgrade: Updating iscsi.foo.smithi086.rsbqwx 2024-04-02T05:21:43.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:43 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:43.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:43 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:21:43.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:43 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:21:43.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:43 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:21:43.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:43 smithi192 ceph-mon[68000]: Deploying daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T05:21:43.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:43 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/1429895911' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:43.687 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:21:43 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:21:43] "GET /metrics HTTP/1.1" 200 38401 "" "Prometheus/2.43.0" 2024-04-02T05:21:43.688 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:43 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:21:43.688 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:43 smithi086 ceph-mon[88839]: from='client.54765 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:21:43.688 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:43 smithi086 ceph-mon[88839]: from='client.64711 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:21:43.688 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:43 smithi086 ceph-mon[88839]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:21:43.688 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:43 smithi086 ceph-mon[88839]: Upgrade: Updating iscsi.foo.smithi086.rsbqwx 2024-04-02T05:21:43.688 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:43 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:43.688 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:43 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:21:43.688 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:43 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:21:43.688 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:43 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:21:43.688 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:43 smithi086 ceph-mon[88839]: Deploying daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T05:21:43.688 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:43 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/1429895911' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:43.689 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:43 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:21:43.689 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:43 smithi086 ceph-mon[86443]: from='client.54765 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:21:43.689 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:43 smithi086 ceph-mon[86443]: from='client.64711 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:21:43.689 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:43 smithi086 ceph-mon[86443]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:21:43.689 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:43 smithi086 ceph-mon[86443]: Upgrade: Updating iscsi.foo.smithi086.rsbqwx 2024-04-02T05:21:43.689 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:43 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:43.689 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:43 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:21:43.689 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:43 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi086.rsbqwx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T05:21:43.689 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:43 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:21:43.689 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:43 smithi086 ceph-mon[86443]: Deploying daemon iscsi.foo.smithi086.rsbqwx on smithi086 2024-04-02T05:21:43.689 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:43 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/1429895911' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:21:44.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:44 smithi192 ceph-mon[68000]: from='client.64717 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:21:44.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:44 smithi192 ceph-mon[68000]: from='client.54789 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:21:44.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:44 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/2174516686' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T05:21:44.684 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:44 smithi086 ceph-mon[88839]: from='client.64717 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:21:44.684 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:44 smithi086 ceph-mon[88839]: from='client.54789 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:21:44.684 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:44 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/2174516686' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T05:21:44.685 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:44 smithi086 ceph-mon[86443]: from='client.64717 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:21:44.685 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:44 smithi086 ceph-mon[86443]: from='client.54789 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:21:44.685 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:44 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/2174516686' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T05:21:45.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:45 smithi192 ceph-mon[68000]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:21:45.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:45 smithi086 ceph-mon[86443]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:21:45.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:45 smithi086 ceph-mon[88839]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:21:47.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:47 smithi192 ceph-mon[68000]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:21:47.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:47 smithi086 ceph-mon[86443]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:21:47.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:47 smithi086 ceph-mon[88839]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:21:49.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:49 smithi192 ceph-mon[68000]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:21:49.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:49 smithi086 ceph-mon[86443]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:21:49.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:49 smithi086 ceph-mon[88839]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:21:51.657 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:51 smithi086 ceph-mon[86443]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:21:51.658 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:51 smithi086 ceph-mon[88839]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:21:51.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:51 smithi192 ceph-mon[68000]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:21:53.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:53 smithi192 ceph-mon[68000]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:21:53.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:53 smithi192 ceph-mon[68000]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:21:53.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:53 smithi086 ceph-mon[88839]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:21:53.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:53 smithi086 ceph-mon[88839]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:21:53.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:53 smithi086 ceph-mon[86443]: from='client.35291 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:21:53.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:53 smithi086 ceph-mon[86443]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:21:53.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:21:53 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:21:53] "GET /metrics HTTP/1.1" 200 38401 "" "Prometheus/2.43.0" 2024-04-02T05:21:54.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:54 smithi086 ceph-mon[86443]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:21:54.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:54 smithi086 ceph-mon[88839]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:21:54.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:54 smithi192 ceph-mon[68000]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:21:55.669 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:55 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:21:55.670 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:55 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:21:55.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:55 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:21:56.556 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:21:56 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:21:56.192+0000 7fe16d7aa700 -1 log_channel(cephadm) log [ERR] : Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi086.rsbqwx on host smithi086 failed. 2024-04-02T05:21:57.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:57 smithi086 ceph-mon[86443]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi086.rsbqwx on host smithi086 failed. 2024-04-02T05:21:57.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:57 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:57.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:57 smithi086 ceph-mon[86443]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 865 B/s rd, 0 op/s 2024-04-02T05:21:57.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:57 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:21:57.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:57 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:21:57.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:57 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:57.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:57 smithi086 ceph-mon[88839]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi086.rsbqwx on host smithi086 failed. 2024-04-02T05:21:57.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:57 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:57.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:57 smithi086 ceph-mon[88839]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 865 B/s rd, 0 op/s 2024-04-02T05:21:57.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:57 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:21:57.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:57 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:21:57.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:57 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:57.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:57 smithi192 ceph-mon[68000]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi086.rsbqwx on host smithi086 failed. 2024-04-02T05:21:57.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:57 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:57.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:57 smithi192 ceph-mon[68000]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 865 B/s rd, 0 op/s 2024-04-02T05:21:57.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:57 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:21:57.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:57 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:21:57.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:57 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:21:58.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:58 smithi086 ceph-mon[86443]: Health check failed: Upgrading daemon iscsi.foo.smithi086.rsbqwx on host smithi086 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-04-02T05:21:58.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:58 smithi086 ceph-mon[88839]: Health check failed: Upgrading daemon iscsi.foo.smithi086.rsbqwx on host smithi086 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-04-02T05:21:58.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:58 smithi192 ceph-mon[68000]: Health check failed: Upgrading daemon iscsi.foo.smithi086.rsbqwx on host smithi086 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-04-02T05:21:59.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:21:59 smithi086 ceph-mon[86443]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T05:21:59.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:21:59 smithi086 ceph-mon[88839]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T05:21:59.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:21:59 smithi192 ceph-mon[68000]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T05:22:01.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:01 smithi086 ceph-mon[86443]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 606 B/s rd, 0 op/s 2024-04-02T05:22:01.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:01 smithi086 ceph-mon[88839]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 606 B/s rd, 0 op/s 2024-04-02T05:22:01.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:01 smithi192 ceph-mon[68000]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 606 B/s rd, 0 op/s 2024-04-02T05:22:03.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:03 smithi086 ceph-mon[86443]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 606 B/s rd, 0 op/s 2024-04-02T05:22:03.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:22:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:22:03] "GET /metrics HTTP/1.1" 200 38406 "" "Prometheus/2.43.0" 2024-04-02T05:22:03.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:03 smithi086 ceph-mon[88839]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 606 B/s rd, 0 op/s 2024-04-02T05:22:03.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:03 smithi192 ceph-mon[68000]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 606 B/s rd, 0 op/s 2024-04-02T05:22:05.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:05 smithi086 ceph-mon[86443]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 606 B/s rd, 0 op/s 2024-04-02T05:22:05.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:05 smithi086 ceph-mon[88839]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 606 B/s rd, 0 op/s 2024-04-02T05:22:05.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:05 smithi192 ceph-mon[68000]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 606 B/s rd, 0 op/s 2024-04-02T05:22:07.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:07 smithi086 ceph-mon[86443]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 173 B/s rd, 0 op/s 2024-04-02T05:22:07.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:07 smithi086 ceph-mon[88839]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 173 B/s rd, 0 op/s 2024-04-02T05:22:07.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:07 smithi192 ceph-mon[68000]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 173 B/s rd, 0 op/s 2024-04-02T05:22:08.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:08 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/2541432789' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T05:22:08.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:08 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/493643457' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1817360796"}]: dispatch 2024-04-02T05:22:08.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:08 smithi086 ceph-mon[86443]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1817360796"}]: dispatch 2024-04-02T05:22:08.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:08 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/2541432789' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T05:22:08.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:08 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/493643457' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1817360796"}]: dispatch 2024-04-02T05:22:08.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:08 smithi086 ceph-mon[88839]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1817360796"}]: dispatch 2024-04-02T05:22:08.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:08 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/2541432789' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T05:22:08.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:08 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/493643457' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1817360796"}]: dispatch 2024-04-02T05:22:08.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:08 smithi192 ceph-mon[68000]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1817360796"}]: dispatch 2024-04-02T05:22:09.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:09 smithi086 ceph-mon[88839]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-02T05:22:09.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:09 smithi086 ceph-mon[88839]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1817360796"}]': finished 2024-04-02T05:22:09.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:09 smithi086 ceph-mon[88839]: osdmap e159: 8 total, 8 up, 8 in 2024-04-02T05:22:09.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:09 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/1114356755' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/911202436"}]: dispatch 2024-04-02T05:22:09.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:09 smithi086 ceph-mon[88839]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/911202436"}]: dispatch 2024-04-02T05:22:09.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:09 smithi086 ceph-mon[86443]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-02T05:22:09.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:09 smithi086 ceph-mon[86443]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1817360796"}]': finished 2024-04-02T05:22:09.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:09 smithi086 ceph-mon[86443]: osdmap e159: 8 total, 8 up, 8 in 2024-04-02T05:22:09.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:09 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/1114356755' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/911202436"}]: dispatch 2024-04-02T05:22:09.584 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:09 smithi086 ceph-mon[86443]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/911202436"}]: dispatch 2024-04-02T05:22:09.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:09 smithi192 ceph-mon[68000]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-02T05:22:09.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:09 smithi192 ceph-mon[68000]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1817360796"}]': finished 2024-04-02T05:22:09.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:09 smithi192 ceph-mon[68000]: osdmap e159: 8 total, 8 up, 8 in 2024-04-02T05:22:09.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:09 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/1114356755' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/911202436"}]: dispatch 2024-04-02T05:22:09.678 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:09 smithi192 ceph-mon[68000]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/911202436"}]: dispatch 2024-04-02T05:22:10.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:10 smithi086 ceph-mon[88839]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/911202436"}]': finished 2024-04-02T05:22:10.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:10 smithi086 ceph-mon[88839]: osdmap e160: 8 total, 8 up, 8 in 2024-04-02T05:22:10.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:10 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/618185157' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1543637491"}]: dispatch 2024-04-02T05:22:10.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:10 smithi086 ceph-mon[88839]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1543637491"}]: dispatch 2024-04-02T05:22:10.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:10 smithi086 ceph-mon[86443]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/911202436"}]': finished 2024-04-02T05:22:10.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:10 smithi086 ceph-mon[86443]: osdmap e160: 8 total, 8 up, 8 in 2024-04-02T05:22:10.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:10 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/618185157' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1543637491"}]: dispatch 2024-04-02T05:22:10.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:10 smithi086 ceph-mon[86443]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1543637491"}]: dispatch 2024-04-02T05:22:10.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:10 smithi192 ceph-mon[68000]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6800/911202436"}]': finished 2024-04-02T05:22:10.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:10 smithi192 ceph-mon[68000]: osdmap e160: 8 total, 8 up, 8 in 2024-04-02T05:22:10.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:10 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/618185157' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1543637491"}]: dispatch 2024-04-02T05:22:10.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:10 smithi192 ceph-mon[68000]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1543637491"}]: dispatch 2024-04-02T05:22:11.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:11 smithi086 ceph-mon[86443]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-04-02T05:22:11.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:11 smithi086 ceph-mon[86443]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1543637491"}]': finished 2024-04-02T05:22:11.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:11 smithi086 ceph-mon[86443]: osdmap e161: 8 total, 8 up, 8 in 2024-04-02T05:22:11.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:11 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:22:11.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:11 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:22:11.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:11 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/3758312347' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/911202436"}]: dispatch 2024-04-02T05:22:11.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:11 smithi086 ceph-mon[88839]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-04-02T05:22:11.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:11 smithi086 ceph-mon[88839]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1543637491"}]': finished 2024-04-02T05:22:11.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:11 smithi086 ceph-mon[88839]: osdmap e161: 8 total, 8 up, 8 in 2024-04-02T05:22:11.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:11 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:22:11.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:11 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:22:11.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:11 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/3758312347' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/911202436"}]: dispatch 2024-04-02T05:22:11.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:11 smithi192 ceph-mon[68000]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-04-02T05:22:11.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:11 smithi192 ceph-mon[68000]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1543637491"}]': finished 2024-04-02T05:22:11.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:11 smithi192 ceph-mon[68000]: osdmap e161: 8 total, 8 up, 8 in 2024-04-02T05:22:11.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:11 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:22:11.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:11 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:22:11.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:11 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/3758312347' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/911202436"}]: dispatch 2024-04-02T05:22:12.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:12 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/3758312347' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/911202436"}]': finished 2024-04-02T05:22:12.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:12 smithi086 ceph-mon[86443]: osdmap e162: 8 total, 8 up, 8 in 2024-04-02T05:22:12.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:12 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/2420637501' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1620818069"}]: dispatch 2024-04-02T05:22:12.586 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:12 smithi086 ceph-mon[86443]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1620818069"}]: dispatch 2024-04-02T05:22:12.586 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:12 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/3758312347' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/911202436"}]': finished 2024-04-02T05:22:12.586 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:12 smithi086 ceph-mon[88839]: osdmap e162: 8 total, 8 up, 8 in 2024-04-02T05:22:12.586 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:12 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/2420637501' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1620818069"}]: dispatch 2024-04-02T05:22:12.586 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:12 smithi086 ceph-mon[88839]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1620818069"}]: dispatch 2024-04-02T05:22:12.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:12 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/3758312347' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:6801/911202436"}]': finished 2024-04-02T05:22:12.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:12 smithi192 ceph-mon[68000]: osdmap e162: 8 total, 8 up, 8 in 2024-04-02T05:22:12.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:12 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/2420637501' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1620818069"}]: dispatch 2024-04-02T05:22:12.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:12 smithi192 ceph-mon[68000]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1620818069"}]: dispatch 2024-04-02T05:22:13.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:13 smithi192 ceph-mon[68000]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:22:13.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:13 smithi192 ceph-mon[68000]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1620818069"}]': finished 2024-04-02T05:22:13.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:13 smithi192 ceph-mon[68000]: osdmap e163: 8 total, 8 up, 8 in 2024-04-02T05:22:13.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:13 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/408833923' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/474070947"}]: dispatch 2024-04-02T05:22:13.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:13 smithi192 ceph-mon[68000]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/474070947"}]: dispatch 2024-04-02T05:22:13.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:22:13 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:22:13] "GET /metrics HTTP/1.1" 200 38485 "" "Prometheus/2.43.0" 2024-04-02T05:22:13.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:13 smithi086 ceph-mon[86443]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:22:13.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:13 smithi086 ceph-mon[86443]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1620818069"}]': finished 2024-04-02T05:22:13.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:13 smithi086 ceph-mon[86443]: osdmap e163: 8 total, 8 up, 8 in 2024-04-02T05:22:13.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:13 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/408833923' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/474070947"}]: dispatch 2024-04-02T05:22:13.834 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:13 smithi086 ceph-mon[86443]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/474070947"}]: dispatch 2024-04-02T05:22:13.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:13 smithi086 ceph-mon[88839]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 265 MiB used, 715 GiB / 715 GiB avail 2024-04-02T05:22:13.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:13 smithi086 ceph-mon[88839]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/1620818069"}]': finished 2024-04-02T05:22:13.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:13 smithi086 ceph-mon[88839]: osdmap e163: 8 total, 8 up, 8 in 2024-04-02T05:22:13.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:13 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/408833923' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/474070947"}]: dispatch 2024-04-02T05:22:13.834 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:13 smithi086 ceph-mon[88839]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/474070947"}]: dispatch 2024-04-02T05:22:13.947 INFO:teuthology.orchestra.run.smithi086.stdout:true 2024-04-02T05:22:14.282 INFO:teuthology.orchestra.run.smithi086.stdout:"Error: UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi086.rsbqwx on host smithi086 failed." 2024-04-02T05:22:14.557 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:14 smithi086 ceph-mon[88839]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/474070947"}]': finished 2024-04-02T05:22:14.558 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:14 smithi086 ceph-mon[88839]: osdmap e164: 8 total, 8 up, 8 in 2024-04-02T05:22:14.558 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:14 smithi086 ceph-mon[86443]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/474070947"}]': finished 2024-04-02T05:22:14.558 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:14 smithi086 ceph-mon[86443]: osdmap e164: 8 total, 8 up, 8 in 2024-04-02T05:22:14.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:14 smithi192 ceph-mon[68000]: from='client.? ' entity='client.iscsi.foo.smithi086.rsbqwx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.86:0/474070947"}]': finished 2024-04-02T05:22:14.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:14 smithi192 ceph-mon[68000]: osdmap e164: 8 total, 8 up, 8 in 2024-04-02T05:22:14.949 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch ps' 2024-04-02T05:22:15.347 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:22:15 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=cleanup t=2024-04-02T05:22:15.062345898Z level=info msg="Completed cleanup jobs" duration=2.244078ms 2024-04-02T05:22:15.347 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:22:15 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=sqlstore.transactions t=2024-04-02T05:22:15.070631911Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-02T05:22:15.569 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:15 smithi086 ceph-mon[88839]: from='client.64801 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:22:15.569 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:15 smithi086 ceph-mon[88839]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:22:15.570 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:15 smithi086 ceph-mon[88839]: from='client.54867 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:22:15.570 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:15 smithi086 ceph-mon[86443]: from='client.64801 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:22:15.570 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:15 smithi086 ceph-mon[86443]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:22:15.570 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:15 smithi086 ceph-mon[86443]: from='client.54867 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:22:15.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:15 smithi192 ceph-mon[68000]: from='client.64801 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:22:15.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:15 smithi192 ceph-mon[68000]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:22:15.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:15 smithi192 ceph-mon[68000]: from='client.54867 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:22:16.942 INFO:teuthology.orchestra.run.smithi086.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T05:22:16.942 INFO:teuthology.orchestra.run.smithi086.stdout:alertmanager.a smithi086 *:9093,9094 running (10m) 72s ago 32m 25.7M - 0.25.0 c8568f914cd2 660ea62bb6b9 2024-04-02T05:22:16.942 INFO:teuthology.orchestra.run.smithi086.stdout:grafana.a smithi192 *:3000 running (10m) 72s ago 31m 75.5M - 9.4.12 2bacad6d85d8 259af9e63aa5 2024-04-02T05:22:16.942 INFO:teuthology.orchestra.run.smithi086.stdout:iscsi.foo.smithi086.rsbqwx smithi086 running (9m) 72s ago 31m 64.5M - 3.5 e1d6a67b021e fe728eb940f7 2024-04-02T05:22:16.942 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.x smithi192 *:8443,9283,8765 running (11m) 72s ago 36m 454M - 19.0.0-1979-g6cec793d 78f830190839 dd1e6d0f0a95 2024-04-02T05:22:16.942 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.y smithi086 *:8443,9283,8765 running (21m) 72s ago 38m 512M - 19.0.0-1979-g6cec793d 78f830190839 3dca54a018ab 2024-04-02T05:22:16.942 INFO:teuthology.orchestra.run.smithi086.stdout:mon.a smithi086 running (8m) 72s ago 38m 59.0M 2048M 19.0.0-1979-g6cec793d 78f830190839 8e478dd4da2d 2024-04-02T05:22:16.942 INFO:teuthology.orchestra.run.smithi086.stdout:mon.b smithi192 running (9m) 72s ago 36m 52.5M 2048M 19.0.0-1979-g6cec793d 78f830190839 4125e997188f 2024-04-02T05:22:16.943 INFO:teuthology.orchestra.run.smithi086.stdout:mon.c smithi086 running (7m) 72s ago 37m 48.7M 2048M 19.0.0-1979-g6cec793d 78f830190839 8180999b3142 2024-04-02T05:22:16.943 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.a smithi086 *:9100 running (11m) 72s ago 32m 15.3M - 1.5.0 0da6a335fe13 05a844569d7b 2024-04-02T05:22:16.943 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.b smithi192 *:9100 running (10m) 72s ago 32m 16.5M - 1.5.0 0da6a335fe13 1a7d6788d58b 2024-04-02T05:22:16.943 INFO:teuthology.orchestra.run.smithi086.stdout:osd.0 smithi086 running (6m) 72s ago 36m 55.4M 2992M 19.0.0-1979-g6cec793d 78f830190839 e521a0ef7daf 2024-04-02T05:22:16.943 INFO:teuthology.orchestra.run.smithi086.stdout:osd.1 smithi086 running (6m) 72s ago 35m 65.1M 2992M 19.0.0-1979-g6cec793d 78f830190839 76fe72844771 2024-04-02T05:22:16.943 INFO:teuthology.orchestra.run.smithi086.stdout:osd.2 smithi086 running (5m) 72s ago 35m 56.8M 2992M 19.0.0-1979-g6cec793d 78f830190839 28594f089a8f 2024-04-02T05:22:16.943 INFO:teuthology.orchestra.run.smithi086.stdout:osd.3 smithi086 running (4m) 72s ago 35m 51.4M 2992M 19.0.0-1979-g6cec793d 78f830190839 6028f927dfd6 2024-04-02T05:22:16.943 INFO:teuthology.orchestra.run.smithi086.stdout:osd.4 smithi192 running (3m) 72s ago 34m 58.2M 3248M 19.0.0-1979-g6cec793d 78f830190839 f9c58b761b62 2024-04-02T05:22:16.943 INFO:teuthology.orchestra.run.smithi086.stdout:osd.5 smithi192 running (3m) 72s ago 34m 56.7M 3248M 19.0.0-1979-g6cec793d 78f830190839 813afd5cf75d 2024-04-02T05:22:16.943 INFO:teuthology.orchestra.run.smithi086.stdout:osd.6 smithi192 running (2m) 72s ago 34m 55.9M 3248M 19.0.0-1979-g6cec793d 78f830190839 d3707b413a43 2024-04-02T05:22:16.943 INFO:teuthology.orchestra.run.smithi086.stdout:osd.7 smithi192 running (2m) 72s ago 33m 56.3M 3248M 19.0.0-1979-g6cec793d 78f830190839 48f6eb4f415f 2024-04-02T05:22:16.943 INFO:teuthology.orchestra.run.smithi086.stdout:prometheus.a smithi192 *:9095 running (10m) 72s ago 33m 72.6M - 2.43.0 a07b618ecd1d 8378fcdb60e3 2024-04-02T05:22:16.943 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi086.bxzxlp smithi086 *:8000 running (82s) 72s ago 31m 76.2M - 19.0.0-1979-g6cec793d 78f830190839 8c2d4c7203c3 2024-04-02T05:22:16.943 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi192.pmlbot smithi192 *:8000 running (77s) 72s ago 31m 75.2M - 19.0.0-1979-g6cec793d 78f830190839 5adcaee89616 2024-04-02T05:22:17.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:17 smithi192 ceph-mon[68000]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 859 B/s rd, 0 op/s 2024-04-02T05:22:17.747 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:17 smithi086 ceph-mon[88839]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 859 B/s rd, 0 op/s 2024-04-02T05:22:17.748 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:17 smithi086 ceph-mon[86443]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 859 B/s rd, 0 op/s 2024-04-02T05:22:17.771 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph versions' 2024-04-02T05:22:18.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:18 smithi086 ceph-mon[86443]: from='client.54873 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:22:18.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:18 smithi086 ceph-mon[86443]: from='client.54810 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:22:18.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:18 smithi086 ceph-mon[88839]: from='client.54873 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:22:18.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:18 smithi086 ceph-mon[88839]: from='client.54810 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:22:18.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:18 smithi192 ceph-mon[68000]: from='client.54873 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:22:18.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:18 smithi192 ceph-mon[68000]: from='client.54810 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:22:19.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:19 smithi086 ceph-mon[88839]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-04-02T05:22:19.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:19 smithi086 ceph-mon[86443]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-04-02T05:22:19.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:19 smithi192 ceph-mon[68000]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-04-02T05:22:19.809 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:22:19.809 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": { 2024-04-02T05:22:19.809 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 3 2024-04-02T05:22:19.810 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:22:19.810 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": { 2024-04-02T05:22:19.810 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 2 2024-04-02T05:22:19.810 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:22:19.810 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": { 2024-04-02T05:22:19.810 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 8 2024-04-02T05:22:19.810 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:22:19.810 INFO:teuthology.orchestra.run.smithi086.stdout: "rgw": { 2024-04-02T05:22:19.810 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 2 2024-04-02T05:22:19.810 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:22:19.810 INFO:teuthology.orchestra.run.smithi086.stdout: "overall": { 2024-04-02T05:22:19.810 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 15 2024-04-02T05:22:19.810 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-04-02T05:22:19.810 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:22:20.519 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:20 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/3931750733' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:22:20.519 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:20 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/3931750733' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:22:20.543 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-04-02T05:22:20.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:20 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/3931750733' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:22:21.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:21 smithi086 ceph-mon[86443]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:22:21.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:21 smithi086 ceph-mon[88839]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:22:21.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:21 smithi192 ceph-mon[68000]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:22:22.261 INFO:teuthology.orchestra.run.smithi086.stdout:wait for servicemap items w/ changing names to refresh 2024-04-02T05:22:22.898 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'sleep 60' 2024-04-02T05:22:23.460 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:23 smithi086 ceph-mon[86443]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T05:22:23.461 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:22:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:22:23] "GET /metrics HTTP/1.1" 200 38485 "" "Prometheus/2.43.0" 2024-04-02T05:22:23.461 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:23 smithi086 ceph-mon[88839]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T05:22:23.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:23 smithi192 ceph-mon[68000]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-02T05:22:25.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:25 smithi192 ceph-mon[68000]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 942 B/s rd, 0 op/s 2024-04-02T05:22:25.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:25 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:22:25.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:25 smithi086 ceph-mon[86443]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 942 B/s rd, 0 op/s 2024-04-02T05:22:25.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:25 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:22:25.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:25 smithi086 ceph-mon[88839]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 942 B/s rd, 0 op/s 2024-04-02T05:22:25.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:25 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:22:27.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:27 smithi192 ceph-mon[68000]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:22:27.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:27 smithi086 ceph-mon[86443]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:22:27.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:27 smithi086 ceph-mon[88839]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:22:28.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:28 smithi192 ceph-mon[68000]: from='client.54810 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:22:28.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:28 smithi086 ceph-mon[86443]: from='client.54810 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:22:28.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:28 smithi086 ceph-mon[88839]: from='client.54810 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:22:29.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:29 smithi192 ceph-mon[68000]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:22:29.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:29 smithi086 ceph-mon[86443]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:22:29.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:29 smithi086 ceph-mon[88839]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:22:31.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:31 smithi192 ceph-mon[68000]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:22:31.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:31 smithi086 ceph-mon[86443]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:22:31.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:31 smithi086 ceph-mon[88839]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:22:33.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:33 smithi192 ceph-mon[68000]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:22:33.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:33 smithi086 ceph-mon[88839]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:22:33.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:33 smithi086 ceph-mon[86443]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:22:33.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:22:33 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:22:33] "GET /metrics HTTP/1.1" 200 38485 "" "Prometheus/2.43.0" 2024-04-02T05:22:35.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:35 smithi192 ceph-mon[68000]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:22:35.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:35 smithi086 ceph-mon[86443]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:22:35.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:35 smithi086 ceph-mon[88839]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:22:37.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:37 smithi192 ceph-mon[68000]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-02T05:22:37.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:37 smithi086 ceph-mon[86443]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-02T05:22:37.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:37 smithi086 ceph-mon[88839]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-02T05:22:38.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:38 smithi192 ceph-mon[68000]: from='client.54810 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:22:38.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:38 smithi086 ceph-mon[86443]: from='client.54810 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:22:38.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:38 smithi086 ceph-mon[88839]: from='client.54810 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:22:39.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:39 smithi192 ceph-mon[68000]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:22:39.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:39 smithi086 ceph-mon[86443]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:22:39.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:39 smithi086 ceph-mon[88839]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:22:40.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:40 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:22:40.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:40 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:22:40.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:40 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:22:41.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:41 smithi192 ceph-mon[68000]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-02T05:22:41.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:41 smithi086 ceph-mon[86443]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-02T05:22:41.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:41 smithi086 ceph-mon[88839]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-02T05:22:42.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:42 smithi192 ceph-mon[68000]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-02T05:22:42.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:42 smithi086 ceph-mon[86443]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-02T05:22:42.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:42 smithi086 ceph-mon[88839]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-02T05:22:43.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:22:43 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:22:43] "GET /metrics HTTP/1.1" 200 38486 "" "Prometheus/2.43.0" 2024-04-02T05:22:45.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:45 smithi086 ceph-mon[88839]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:22:45.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:45 smithi086 ceph-mon[86443]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:22:45.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:45 smithi192 ceph-mon[68000]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:22:47.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:47 smithi086 ceph-mon[88839]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:22:47.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:47 smithi086 ceph-mon[86443]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:22:47.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:47 smithi192 ceph-mon[68000]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:22:48.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:48 smithi086 ceph-mon[88839]: from='client.54810 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:22:48.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:48 smithi086 ceph-mon[86443]: from='client.54810 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:22:48.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:48 smithi192 ceph-mon[68000]: from='client.54810 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:22:49.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:49 smithi086 ceph-mon[88839]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:22:49.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:49 smithi086 ceph-mon[86443]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:22:49.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:49 smithi192 ceph-mon[68000]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:22:51.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:51 smithi086 ceph-mon[88839]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:22:51.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:51 smithi086 ceph-mon[86443]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:22:51.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:51 smithi192 ceph-mon[68000]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:22:53.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:53 smithi086 ceph-mon[86443]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:22:53.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:22:53 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:22:53] "GET /metrics HTTP/1.1" 200 38486 "" "Prometheus/2.43.0" 2024-04-02T05:22:53.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:53 smithi086 ceph-mon[88839]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:22:53.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:53 smithi192 ceph-mon[68000]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:22:55.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:55 smithi086 ceph-mon[86443]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:22:55.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:55 smithi086 ceph-mon[88839]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:22:55.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:55 smithi192 ceph-mon[68000]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:22:56.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:56 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:22:56.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:56 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:22:56.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:56 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:22:57.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:57 smithi086 ceph-mon[86443]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:22:57.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:57 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:22:57.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:57 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:22:57.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:57 smithi086 ceph-mon[86443]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:22:57.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:57 smithi086 ceph-mon[88839]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:22:57.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:57 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:22:57.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:57 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:22:57.584 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:57 smithi086 ceph-mon[88839]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:22:57.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:57 smithi192 ceph-mon[68000]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:22:57.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:57 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T05:22:57.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:57 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T05:22:57.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:57 smithi192 ceph-mon[68000]: from='mgr.25801 ' entity='mgr.y' 2024-04-02T05:22:58.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:58 smithi086 ceph-mon[86443]: from='client.54810 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:22:58.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:58 smithi086 ceph-mon[88839]: from='client.54810 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:22:58.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:58 smithi192 ceph-mon[68000]: from='client.54810 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:22:59.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:22:59 smithi086 ceph-mon[86443]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:22:59.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:22:59 smithi086 ceph-mon[88839]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:22:59.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:22:59 smithi192 ceph-mon[68000]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:23:01.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:01 smithi086 ceph-mon[86443]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:01.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:01 smithi086 ceph-mon[88839]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:01.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:01 smithi192 ceph-mon[68000]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:03.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:03 smithi086 ceph-mon[86443]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:03.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:03 smithi086 ceph-mon[88839]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:03.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:23:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:23:03] "GET /metrics HTTP/1.1" 200 38482 "" "Prometheus/2.43.0" 2024-04-02T05:23:03.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:03 smithi192 ceph-mon[68000]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:05.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:05 smithi086 ceph-mon[88839]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:23:05.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:05 smithi086 ceph-mon[86443]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:23:05.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:05 smithi192 ceph-mon[68000]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:23:07.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:07 smithi086 ceph-mon[86443]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:07.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:07 smithi086 ceph-mon[88839]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:07.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:07 smithi192 ceph-mon[68000]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:08.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:08 smithi086 ceph-mon[86443]: from='client.54810 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:23:08.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:08 smithi086 ceph-mon[88839]: from='client.54810 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:23:08.632 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:08 smithi192 ceph-mon[68000]: from='client.54810 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:23:09.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:09 smithi086 ceph-mon[86443]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:23:09.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:09 smithi086 ceph-mon[88839]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:23:09.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:09 smithi192 ceph-mon[68000]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:23:10.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:10 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:23:10.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:10 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:23:10.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:10 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:23:11.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:11 smithi192 ceph-mon[68000]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-02T05:23:11.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:11 smithi086 ceph-mon[86443]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-02T05:23:11.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:11 smithi086 ceph-mon[88839]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-02T05:23:13.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:13 smithi192 ceph-mon[68000]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-02T05:23:13.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:13 smithi086 ceph-mon[88839]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-02T05:23:13.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:13 smithi086 ceph-mon[86443]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-02T05:23:13.833 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:23:13 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:23:13] "GET /metrics HTTP/1.1" 200 38484 "" "Prometheus/2.43.0" 2024-04-02T05:23:15.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:15 smithi192 ceph-mon[68000]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:23:15.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:15 smithi086 ceph-mon[86443]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:23:15.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:15 smithi086 ceph-mon[88839]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:23:17.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:17 smithi086 ceph-mon[86443]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-02T05:23:17.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:17 smithi086 ceph-mon[88839]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-02T05:23:17.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:17 smithi192 ceph-mon[68000]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-02T05:23:18.427 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:18 smithi192 ceph-mon[68000]: from='client.54810 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:23:18.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:18 smithi086 ceph-mon[86443]: from='client.54810 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:23:18.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:18 smithi086 ceph-mon[88839]: from='client.54810 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:23:19.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:19 smithi192 ceph-mon[68000]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:23:19.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:19 smithi086 ceph-mon[86443]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:23:19.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:19 smithi086 ceph-mon[88839]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:23:21.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:21 smithi192 ceph-mon[68000]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:21.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:21 smithi086 ceph-mon[86443]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:21.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:21 smithi086 ceph-mon[88839]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:23.634 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:23 smithi086 ceph-mon[86443]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:23.635 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:23 smithi086 ceph-mon[88839]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:23.635 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:23:23 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:23:23] "GET /metrics HTTP/1.1" 200 38484 "" "Prometheus/2.43.0" 2024-04-02T05:23:23.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:23 smithi192 ceph-mon[68000]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:25.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:25 smithi192 ceph-mon[68000]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:23:25.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:25 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:23:25.701 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:25 smithi086 ceph-mon[86443]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:23:25.702 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:25 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:23:25.702 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:25 smithi086 ceph-mon[88839]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:23:25.702 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:25 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:23:25.730 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch ps' 2024-04-02T05:23:27.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:27 smithi086 ceph-mon[86443]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:27.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:27 smithi086 ceph-mon[88839]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:27.659 INFO:teuthology.orchestra.run.smithi086.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T05:23:27.659 INFO:teuthology.orchestra.run.smithi086.stdout:alertmanager.a smithi086 *:9093,9094 running (11m) 2m ago 33m 25.7M - 0.25.0 c8568f914cd2 660ea62bb6b9 2024-04-02T05:23:27.659 INFO:teuthology.orchestra.run.smithi086.stdout:grafana.a smithi192 *:3000 running (11m) 2m ago 33m 75.5M - 9.4.12 2bacad6d85d8 259af9e63aa5 2024-04-02T05:23:27.659 INFO:teuthology.orchestra.run.smithi086.stdout:iscsi.foo.smithi086.rsbqwx smithi086 running (10m) 2m ago 32m 64.5M - 3.5 e1d6a67b021e fe728eb940f7 2024-04-02T05:23:27.659 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.x smithi192 *:8443,9283,8765 running (12m) 2m ago 37m 454M - 19.0.0-1979-g6cec793d 78f830190839 dd1e6d0f0a95 2024-04-02T05:23:27.660 INFO:teuthology.orchestra.run.smithi086.stdout:mgr.y smithi086 *:8443,9283,8765 running (22m) 2m ago 39m 512M - 19.0.0-1979-g6cec793d 78f830190839 3dca54a018ab 2024-04-02T05:23:27.660 INFO:teuthology.orchestra.run.smithi086.stdout:mon.a smithi086 running (9m) 2m ago 39m 59.0M 2048M 19.0.0-1979-g6cec793d 78f830190839 8e478dd4da2d 2024-04-02T05:23:27.660 INFO:teuthology.orchestra.run.smithi086.stdout:mon.b smithi192 running (10m) 2m ago 38m 52.5M 2048M 19.0.0-1979-g6cec793d 78f830190839 4125e997188f 2024-04-02T05:23:27.660 INFO:teuthology.orchestra.run.smithi086.stdout:mon.c smithi086 running (9m) 2m ago 38m 48.7M 2048M 19.0.0-1979-g6cec793d 78f830190839 8180999b3142 2024-04-02T05:23:27.660 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.a smithi086 *:9100 running (12m) 2m ago 34m 15.3M - 1.5.0 0da6a335fe13 05a844569d7b 2024-04-02T05:23:27.660 INFO:teuthology.orchestra.run.smithi086.stdout:node-exporter.b smithi192 *:9100 running (12m) 2m ago 34m 16.5M - 1.5.0 0da6a335fe13 1a7d6788d58b 2024-04-02T05:23:27.660 INFO:teuthology.orchestra.run.smithi086.stdout:osd.0 smithi086 running (7m) 2m ago 37m 55.4M 2992M 19.0.0-1979-g6cec793d 78f830190839 e521a0ef7daf 2024-04-02T05:23:27.660 INFO:teuthology.orchestra.run.smithi086.stdout:osd.1 smithi086 running (7m) 2m ago 37m 65.1M 2992M 19.0.0-1979-g6cec793d 78f830190839 76fe72844771 2024-04-02T05:23:27.660 INFO:teuthology.orchestra.run.smithi086.stdout:osd.2 smithi086 running (6m) 2m ago 36m 56.8M 2992M 19.0.0-1979-g6cec793d 78f830190839 28594f089a8f 2024-04-02T05:23:27.660 INFO:teuthology.orchestra.run.smithi086.stdout:osd.3 smithi086 running (5m) 2m ago 36m 51.4M 2992M 19.0.0-1979-g6cec793d 78f830190839 6028f927dfd6 2024-04-02T05:23:27.660 INFO:teuthology.orchestra.run.smithi086.stdout:osd.4 smithi192 running (4m) 2m ago 36m 58.2M 3248M 19.0.0-1979-g6cec793d 78f830190839 f9c58b761b62 2024-04-02T05:23:27.660 INFO:teuthology.orchestra.run.smithi086.stdout:osd.5 smithi192 running (4m) 2m ago 35m 56.7M 3248M 19.0.0-1979-g6cec793d 78f830190839 813afd5cf75d 2024-04-02T05:23:27.660 INFO:teuthology.orchestra.run.smithi086.stdout:osd.6 smithi192 running (3m) 2m ago 35m 55.9M 3248M 19.0.0-1979-g6cec793d 78f830190839 d3707b413a43 2024-04-02T05:23:27.660 INFO:teuthology.orchestra.run.smithi086.stdout:osd.7 smithi192 running (3m) 2m ago 34m 56.3M 3248M 19.0.0-1979-g6cec793d 78f830190839 48f6eb4f415f 2024-04-02T05:23:27.661 INFO:teuthology.orchestra.run.smithi086.stdout:prometheus.a smithi192 *:9095 running (11m) 2m ago 34m 72.6M - 2.43.0 a07b618ecd1d 8378fcdb60e3 2024-04-02T05:23:27.661 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi086.bxzxlp smithi086 *:8000 running (2m) 2m ago 32m 76.2M - 19.0.0-1979-g6cec793d 78f830190839 8c2d4c7203c3 2024-04-02T05:23:27.661 INFO:teuthology.orchestra.run.smithi086.stdout:rgw.foo.smithi192.pmlbot smithi192 *:8000 running (2m) 2m ago 32m 75.2M - 19.0.0-1979-g6cec793d 78f830190839 5adcaee89616 2024-04-02T05:23:27.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:27 smithi192 ceph-mon[68000]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:28.372 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph versions' 2024-04-02T05:23:28.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:28 smithi192 ceph-mon[68000]: from='client.54810 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:23:28.681 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:28 smithi086 ceph-mon[86443]: from='client.54810 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:23:28.681 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:28 smithi086 ceph-mon[88839]: from='client.54810 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:23:29.629 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:29 smithi086 ceph-mon[88839]: from='client.64825 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:23:29.629 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:29 smithi086 ceph-mon[88839]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:23:29.629 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:29 smithi086 ceph-mon[86443]: from='client.64825 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:23:29.630 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:29 smithi086 ceph-mon[86443]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:23:29.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:29 smithi192 ceph-mon[68000]: from='client.64825 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:23:29.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:29 smithi192 ceph-mon[68000]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:23:30.339 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:23:30.339 INFO:teuthology.orchestra.run.smithi086.stdout: "mon": { 2024-04-02T05:23:30.339 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 3 2024-04-02T05:23:30.340 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:23:30.340 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr": { 2024-04-02T05:23:30.340 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 2 2024-04-02T05:23:30.340 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:23:30.340 INFO:teuthology.orchestra.run.smithi086.stdout: "osd": { 2024-04-02T05:23:30.340 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 8 2024-04-02T05:23:30.340 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:23:30.340 INFO:teuthology.orchestra.run.smithi086.stdout: "rgw": { 2024-04-02T05:23:30.340 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 2 2024-04-02T05:23:30.340 INFO:teuthology.orchestra.run.smithi086.stdout: }, 2024-04-02T05:23:30.340 INFO:teuthology.orchestra.run.smithi086.stdout: "overall": { 2024-04-02T05:23:30.340 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)": 15 2024-04-02T05:23:30.340 INFO:teuthology.orchestra.run.smithi086.stdout: } 2024-04-02T05:23:30.341 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:23:30.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:30 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/3511524258' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:23:30.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:30 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/3511524258' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:23:30.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:30 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/3511524258' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:23:31.247 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch upgrade status' 2024-04-02T05:23:31.556 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:31 smithi086 ceph-mon[88839]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:31.557 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:31 smithi086 ceph-mon[86443]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:31.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:31 smithi192 ceph-mon[68000]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:33.158 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:23:33.158 INFO:teuthology.orchestra.run.smithi086.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", 2024-04-02T05:23:33.158 INFO:teuthology.orchestra.run.smithi086.stdout: "in_progress": true, 2024-04-02T05:23:33.158 INFO:teuthology.orchestra.run.smithi086.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-02T05:23:33.158 INFO:teuthology.orchestra.run.smithi086.stdout: "services_complete": [ 2024-04-02T05:23:33.158 INFO:teuthology.orchestra.run.smithi086.stdout: "rgw", 2024-04-02T05:23:33.158 INFO:teuthology.orchestra.run.smithi086.stdout: "mon", 2024-04-02T05:23:33.158 INFO:teuthology.orchestra.run.smithi086.stdout: "osd", 2024-04-02T05:23:33.158 INFO:teuthology.orchestra.run.smithi086.stdout: "mgr" 2024-04-02T05:23:33.158 INFO:teuthology.orchestra.run.smithi086.stdout: ], 2024-04-02T05:23:33.158 INFO:teuthology.orchestra.run.smithi086.stdout: "progress": "15/21 daemons upgraded", 2024-04-02T05:23:33.159 INFO:teuthology.orchestra.run.smithi086.stdout: "message": "Error: UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi086.rsbqwx on host smithi086 failed.", 2024-04-02T05:23:33.159 INFO:teuthology.orchestra.run.smithi086.stdout: "is_paused": true 2024-04-02T05:23:33.159 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:23:33.483 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:33 smithi086 ceph-mon[88839]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:33.483 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:33 smithi086 ceph-mon[86443]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:33.483 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:23:33 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:23:33] "GET /metrics HTTP/1.1" 200 38484 "" "Prometheus/2.43.0" 2024-04-02T05:23:33.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:33 smithi192 ceph-mon[68000]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:33.773 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph health detail' 2024-04-02T05:23:34.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:34 smithi192 ceph-mon[68000]: from='client.54897 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:23:34.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:34 smithi086 ceph-mon[86443]: from='client.54897 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:23:34.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:34 smithi086 ceph-mon[88839]: from='client.54897 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:23:35.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:35 smithi192 ceph-mon[68000]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:23:35.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:35 smithi086 ceph-mon[86443]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:23:35.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:35 smithi086 ceph-mon[88839]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:23:35.893 INFO:teuthology.orchestra.run.smithi086.stdout:HEALTH_WARN Upgrading daemon iscsi.foo.smithi086.rsbqwx on host smithi086 failed. 2024-04-02T05:23:35.894 INFO:teuthology.orchestra.run.smithi086.stdout:[WRN] UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi086.rsbqwx on host smithi086 failed. 2024-04-02T05:23:35.894 INFO:teuthology.orchestra.run.smithi086.stdout: Upgrade daemon: iscsi.foo.smithi086.rsbqwx: cephadm exited with an error code: 1, stderr: Redeploy daemon iscsi.foo.smithi086.rsbqwx ... 2024-04-02T05:23:35.894 INFO:teuthology.orchestra.run.smithi086.stdout:Creating ceph-iscsi config... 2024-04-02T05:23:35.894 INFO:teuthology.orchestra.run.smithi086.stdout:Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/iscsi-gateway.cfg 2024-04-02T05:23:35.894 INFO:teuthology.orchestra.run.smithi086.stdout:Write file: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/iscsi.foo.smithi086.rsbqwx/tcmu-runner-entrypoint.sh 2024-04-02T05:23:35.894 INFO:teuthology.orchestra.run.smithi086.stdout:Failed to trim old cgroups /sys/fs/cgroup/system.slice/system-ceph\x2d8862e8fe\x2df0ab\x2d11ee\x2db647\x2dcb9ed24678a4.slice/ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service 2024-04-02T05:23:35.894 INFO:teuthology.orchestra.run.smithi086.stdout:Non-zero exit code 1 from systemctl start ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx 2024-04-02T05:23:35.894 INFO:teuthology.orchestra.run.smithi086.stdout:systemctl: stderr Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:23:35.894 INFO:teuthology.orchestra.run.smithi086.stdout:systemctl: stderr See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:23:35.894 INFO:teuthology.orchestra.run.smithi086.stdout:Traceback (most recent call last): 2024-04-02T05:23:35.894 INFO:teuthology.orchestra.run.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-02T05:23:35.894 INFO:teuthology.orchestra.run.smithi086.stdout: return _run_code(code, main_globals, None, 2024-04-02T05:23:35.894 INFO:teuthology.orchestra.run.smithi086.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-02T05:23:35.894 INFO:teuthology.orchestra.run.smithi086.stdout: exec(code, run_globals) 2024-04-02T05:23:35.895 INFO:teuthology.orchestra.run.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5528, in 2024-04-02T05:23:35.895 INFO:teuthology.orchestra.run.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 5516, in main 2024-04-02T05:23:35.895 INFO:teuthology.orchestra.run.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3040, in command_deploy_from 2024-04-02T05:23:35.895 INFO:teuthology.orchestra.run.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3075, in _common_deploy 2024-04-02T05:23:35.895 INFO:teuthology.orchestra.run.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 3095, in _deploy_daemon_container 2024-04-02T05:23:35.895 INFO:teuthology.orchestra.run.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1031, in deploy_daemon 2024-04-02T05:23:35.895 INFO:teuthology.orchestra.run.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/__main__.py", line 1167, in deploy_daemon_units 2024-04-02T05:23:35.895 INFO:teuthology.orchestra.run.smithi086.stdout: File "/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/cephadm.7479cba0ff39d205770eedc6aeba419fe4e74674ec810f20c21b0068d4c8ea9a/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-02T05:23:35.895 INFO:teuthology.orchestra.run.smithi086.stdout:RuntimeError: Failed command: systemctl start ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx: Job for ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service failed because the control process exited with error code. 2024-04-02T05:23:35.895 INFO:teuthology.orchestra.run.smithi086.stdout:See "systemctl status ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" and "journalctl -xeu ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@iscsi.foo.smithi086.rsbqwx.service" for details. 2024-04-02T05:23:36.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:36 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/2824119047' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T05:23:36.696 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:36 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/2824119047' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T05:23:36.696 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:36 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/2824119047' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T05:23:36.720 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-04-02T05:23:37.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:37 smithi086 ceph-mon[86443]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:37.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:37 smithi086 ceph-mon[88839]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:37.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:37 smithi192 ceph-mon[68000]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:38.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:38 smithi192 ceph-mon[68000]: from='client.54810 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:23:38.833 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:38 smithi086 ceph-mon[86443]: from='client.54810 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:23:38.833 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:38 smithi086 ceph-mon[88839]: from='client.54810 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:23:38.874 INFO:teuthology.orchestra.run.smithi086.stdout:true 2024-04-02T05:23:39.559 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:39 smithi086 ceph-mon[88839]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:23:39.560 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:39 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/2988945877' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:23:39.560 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:39 smithi086 ceph-mon[86443]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:23:39.560 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:39 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/2988945877' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:23:39.585 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-04-02T05:23:39.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:39 smithi192 ceph-mon[68000]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:23:39.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:39 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/2988945877' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:23:40.557 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:40 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:23:40.558 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:40 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:23:40.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:40 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:23:41.582 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:41 smithi086 ceph-mon[86443]: pgmap v341: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:41.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:41 smithi086 ceph-mon[88839]: pgmap v341: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:41.597 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph version 19.0.0-1979-g6cec793d (6cec793d9073f0594e2364b9b687ecaf22424cca) squid (dev)" 2024-04-02T05:23:41.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:41 smithi192 ceph-mon[68000]: pgmap v341: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:42.231 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch ls | grep '"'"'^osd '"'"'' 2024-04-02T05:23:42.537 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:42 smithi086 ceph-mon[86443]: from='client.? 172.21.15.86:0/2627588464' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:23:42.538 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:42 smithi086 ceph-mon[88839]: from='client.? 172.21.15.86:0/2627588464' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:23:42.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:42 smithi192 ceph-mon[68000]: from='client.? 172.21.15.86:0/2627588464' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T05:23:43.663 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:43 smithi086 ceph-mon[86443]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:43.664 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:43 smithi086 ceph-mon[88839]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:43.664 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:23:43 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:23:43] "GET /metrics HTTP/1.1" 200 38484 "" "Prometheus/2.43.0" 2024-04-02T05:23:43.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:43 smithi192 ceph-mon[68000]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:44.215 INFO:teuthology.orchestra.run.smithi086.stdout:osd 8 2m ago - 2024-04-02T05:23:44.827 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:44 smithi086 ceph-mon[86443]: from='client.64861 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:23:44.828 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:44 smithi086 ceph-mon[86443]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:23:44.828 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:44 smithi086 ceph-mon[88839]: from='client.64861 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:23:44.828 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:44 smithi086 ceph-mon[88839]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:23:44.850 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-02T05:23:44.859 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi086.front.sepia.ceph.com 2024-04-02T05:23:44.859 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- bash -c 'ceph orch upgrade ls' 2024-04-02T05:23:44.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:44 smithi192 ceph-mon[68000]: from='client.64861 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:23:44.927 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:44 smithi192 ceph-mon[68000]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:23:47.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:47 smithi086 ceph-mon[86443]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:47.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:47 smithi086 ceph-mon[88839]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:47.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:47 smithi192 ceph-mon[68000]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:48.119 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-02T05:23:48.119 INFO:teuthology.orchestra.run.smithi086.stdout: "image": "quay.io/ceph/ceph", 2024-04-02T05:23:48.119 INFO:teuthology.orchestra.run.smithi086.stdout: "registry": "quay.io", 2024-04-02T05:23:48.119 INFO:teuthology.orchestra.run.smithi086.stdout: "bare_image": "ceph/ceph", 2024-04-02T05:23:48.119 INFO:teuthology.orchestra.run.smithi086.stdout: "versions": [] 2024-04-02T05:23:48.119 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-02T05:23:48.449 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:48 smithi086 ceph-mon[88839]: from='client.54927 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:23:48.449 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:48 smithi086 ceph-mon[88839]: from='client.54810 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:23:48.450 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:48 smithi086 ceph-mon[86443]: from='client.54927 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:23:48.450 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:48 smithi086 ceph-mon[86443]: from='client.54810 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:23:48.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:48 smithi192 ceph-mon[68000]: from='client.54927 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:23:48.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:48 smithi192 ceph-mon[68000]: from='client.54810 -' entity='client.iscsi.foo.smithi086.rsbqwx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T05:23:48.759 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- bash -c 'ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0' 2024-04-02T05:23:49.333 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:49 smithi086 ceph-mon[86443]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:23:49.333 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:49 smithi086 ceph-mon[88839]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:23:49.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:49 smithi192 ceph-mon[68000]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:23:51.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:51 smithi086 ceph-mon[86443]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:51.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:51 smithi086 ceph-mon[88839]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:51.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:51 smithi192 ceph-mon[68000]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:51.953 INFO:teuthology.orchestra.run.smithi086.stdout: "16.2.0", 2024-04-02T05:23:52.537 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:52 smithi086 ceph-mon[86443]: from='client.54933 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:23:52.537 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:52 smithi086 ceph-mon[88839]: from='client.54933 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:23:52.564 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 -- bash -c 'ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2' 2024-04-02T05:23:52.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:52 smithi192 ceph-mon[68000]: from='client.54933 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:23:53.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:53 smithi086 ceph-mon[86443]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:53.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:53 smithi086 ceph-mon[88839]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:53.583 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:23:53 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: ::ffff:172.21.15.192 - - [02/Apr/2024:05:23:53] "GET /metrics HTTP/1.1" 200 38484 "" "Prometheus/2.43.0" 2024-04-02T05:23:53.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:53 smithi192 ceph-mon[68000]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T05:23:55.567 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:55 smithi086 ceph-mon[88839]: pgmap v348: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:23:55.567 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:55 smithi086 ceph-mon[86443]: pgmap v348: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:23:55.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:55 smithi192 ceph-mon[68000]: pgmap v348: 161 pgs: 161 active+clean; 585 KiB data, 266 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T05:23:55.682 INFO:teuthology.orchestra.run.smithi086.stdout: "v16.2.2", 2024-04-02T05:23:55.682 INFO:teuthology.orchestra.run.smithi086.stdout: "v16.2.2-20210505", 2024-04-02T05:23:56.300 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-04-02T05:23:56.311 INFO:tasks.cephadm:Teardown begin 2024-04-02T05:23:56.311 DEBUG:teuthology.orchestra.run.smithi086:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-02T05:23:56.343 DEBUG:teuthology.orchestra.run.smithi192:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-02T05:23:56.380 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-04-02T05:23:56.380 DEBUG:teuthology.orchestra.run.smithi086:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-02T05:23:56.398 DEBUG:teuthology.orchestra.run.smithi192:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-02T05:23:56.436 INFO:tasks.cephadm:Stopping all daemons... 2024-04-02T05:23:56.436 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-04-02T05:23:56.437 DEBUG:teuthology.orchestra.run.smithi086:> sudo systemctl stop ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mon.a 2024-04-02T05:23:56.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:56 smithi086 ceph-mon[86443]: from='client.54939 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:23:56.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:56 smithi086 ceph-mon[86443]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:23:56.583 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:56 smithi086 systemd[1]: Stopping Ceph mon.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:23:56.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:56 smithi086 ceph-mon[88839]: from='client.54939 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:23:56.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:56 smithi086 ceph-mon[88839]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:23:56.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:56 smithi192 ceph-mon[68000]: from='client.54939 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T05:23:56.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:56 smithi192 ceph-mon[68000]: from='mgr.25801 172.21.15.86:0/1091400905' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T05:23:57.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:56 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-a[86439]: 2024-04-02T05:23:56.745+0000 7f9d6297b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T05:23:57.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:56 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-a[86439]: 2024-04-02T05:23:56.745+0000 7f9d6297b700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-04-02T05:23:57.083 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:56 smithi086 podman[129997]: 2024-04-02 05:23:56.79710593 +0000 UTC m=+0.180058937 container died 8e478dd4da2dbd4ceb762aa4372bed999d44e3977f4dd14fd300b2ada2762906 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-a, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-6cec793) 2024-04-02T05:23:57.423 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:57 smithi086 podman[129997]: 2024-04-02 05:23:57.170413197 +0000 UTC m=+0.553366210 container cleanup 8e478dd4da2dbd4ceb762aa4372bed999d44e3977f4dd14fd300b2ada2762906 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-a, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=squid-6cec793, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2) 2024-04-02T05:23:57.423 INFO:journalctl@ceph.mon.a.smithi086.stdout:Apr 02 05:23:57 smithi086 bash[129997]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-a 2024-04-02T05:23:57.550 DEBUG:teuthology.orchestra.run.smithi086:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mon.a.service' 2024-04-02T05:23:57.600 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T05:23:57.600 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-04-02T05:23:57.600 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-04-02T05:23:57.600 DEBUG:teuthology.orchestra.run.smithi086:> sudo systemctl stop ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mon.c 2024-04-02T05:23:57.832 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:57 smithi086 systemd[1]: Stopping Ceph mon.c for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:23:58.278 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:57 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-c[88834]: 2024-04-02T05:23:57.875+0000 7f193a4d6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T05:23:58.278 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:57 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-c[88834]: 2024-04-02T05:23:57.875+0000 7f193a4d6700 -1 mon.c@1(peon) e4 *** Got Signal Terminated *** 2024-04-02T05:23:58.279 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:57 smithi086 podman[130214]: 2024-04-02 05:23:57.988146327 +0000 UTC m=+0.206697413 container died 8180999b3142bc0604f44ab3bc63470afcb15ad9f18f162b326a462deb1d619d (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-c, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-6cec793, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-04-02T05:23:58.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:58 smithi086 podman[130214]: 2024-04-02 05:23:58.367192294 +0000 UTC m=+0.585743362 container cleanup 8180999b3142bc0604f44ab3bc63470afcb15ad9f18f162b326a462deb1d619d (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-c, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=squid-6cec793, org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-04-02T05:23:58.583 INFO:journalctl@ceph.mon.c.smithi086.stdout:Apr 02 05:23:58 smithi086 bash[130214]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-c 2024-04-02T05:23:58.785 DEBUG:teuthology.orchestra.run.smithi086:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mon.c.service' 2024-04-02T05:23:58.823 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T05:23:58.823 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-04-02T05:23:58.823 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-04-02T05:23:58.824 DEBUG:teuthology.orchestra.run.smithi192:> sudo systemctl stop ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mon.b 2024-04-02T05:23:59.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:58 smithi192 systemd[1]: Stopping Ceph mon.b for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:23:59.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:59 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-b[67996]: 2024-04-02T05:23:59.223+0000 7fe3968b4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T05:23:59.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:59 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-b[67996]: 2024-04-02T05:23:59.223+0000 7fe3968b4700 -1 mon.b@2(peon) e4 *** Got Signal Terminated *** 2024-04-02T05:23:59.677 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:59 smithi192 podman[101298]: 2024-04-02 05:23:59.459246942 +0000 UTC m=+0.436901102 container died 4125e997188f34942ad2bc582ba763caca0318f503a79e2c8df6530d0b6f3c84 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-b, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=squid-6cec793, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-02T05:24:00.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:59 smithi192 podman[101298]: 2024-04-02 05:23:59.952414942 +0000 UTC m=+0.930069093 container cleanup 4125e997188f34942ad2bc582ba763caca0318f503a79e2c8df6530d0b6f3c84 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-b, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240326, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-6cec793) 2024-04-02T05:24:00.177 INFO:journalctl@ceph.mon.b.smithi192.stdout:Apr 02 05:23:59 smithi192 bash[101298]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mon-b 2024-04-02T05:24:00.422 DEBUG:teuthology.orchestra.run.smithi192:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mon.b.service' 2024-04-02T05:24:00.468 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T05:24:00.468 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-04-02T05:24:00.469 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-04-02T05:24:00.469 DEBUG:teuthology.orchestra.run.smithi086:> sudo systemctl stop ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mgr.y 2024-04-02T05:24:00.798 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:24:00 smithi086 systemd[1]: Stopping Ceph mgr.y for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:24:01.083 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:24:00 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:24:00.795+0000 7fe1db87b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T05:24:01.083 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:24:00 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y[62641]: 2024-04-02T05:24:00.795+0000 7fe1db87b700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-04-02T05:24:01.083 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:24:00 smithi086 podman[130334]: 2024-04-02 05:24:00.827393425 +0000 UTC m=+0.168539303 container died 3dca54a018ab67df0860f4faa8227434aaeeb9e9d474a9a73c5988c7f830e60d (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=squid-6cec793) 2024-04-02T05:24:01.450 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:24:01 smithi086 podman[130334]: 2024-04-02 05:24:01.17944819 +0000 UTC m=+0.520594063 container cleanup 3dca54a018ab67df0860f4faa8227434aaeeb9e9d474a9a73c5988c7f830e60d (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y, ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-6cec793, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-02T05:24:01.450 INFO:journalctl@ceph.mgr.y.smithi086.stdout:Apr 02 05:24:01 smithi086 bash[130334]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-y 2024-04-02T05:24:01.602 DEBUG:teuthology.orchestra.run.smithi086:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mgr.y.service' 2024-04-02T05:24:01.640 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T05:24:01.640 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-04-02T05:24:01.640 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-04-02T05:24:01.640 DEBUG:teuthology.orchestra.run.smithi192:> sudo systemctl stop ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mgr.x 2024-04-02T05:24:01.927 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:24:01 smithi192 systemd[1]: Stopping Ceph mgr.x for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:24:02.367 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:24:01 smithi192 podman[101419]: 2024-04-02 05:24:01.977408844 +0000 UTC m=+0.147822986 container died dd1e6d0f0a95fe679c7b415b6610151be8c3934405421ba58a2ffec5cc2cb32b (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=squid-6cec793, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240326) 2024-04-02T05:24:02.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:24:02 smithi192 podman[101419]: 2024-04-02 05:24:02.522395633 +0000 UTC m=+0.692809783 container cleanup dd1e6d0f0a95fe679c7b415b6610151be8c3934405421ba58a2ffec5cc2cb32b (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, RELEASE=squid-6cec793, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-04-02T05:24:02.677 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:24:02 smithi192 bash[101419]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x 2024-04-02T05:24:02.946 DEBUG:teuthology.orchestra.run.smithi192:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mgr.x.service' 2024-04-02T05:24:02.970 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:24:02 smithi192 podman[101433]: 2024-04-02 05:24:02.715734545 +0000 UTC m=+0.736544304 container remove dd1e6d0f0a95fe679c7b415b6610151be8c3934405421ba58a2ffec5cc2cb32b (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-mgr-x, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=squid-6cec793, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2) 2024-04-02T05:24:02.970 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:24:02 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-02T05:24:02.970 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:24:02 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mgr.x.service: Failed with result 'exit-code'. 2024-04-02T05:24:02.970 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:24:02 smithi192 systemd[1]: Stopped Ceph mgr.x for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:24:02.970 INFO:journalctl@ceph.mgr.x.smithi192.stdout:Apr 02 05:24:02 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@mgr.x.service: Consumed 22.774s CPU time. 2024-04-02T05:24:02.983 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T05:24:02.983 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-04-02T05:24:02.983 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-04-02T05:24:02.984 DEBUG:teuthology.orchestra.run.smithi086:> sudo systemctl stop ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.0 2024-04-02T05:24:03.286 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:24:03 smithi086 systemd[1]: Stopping Ceph osd.0 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:24:03.582 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:24:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0[95933]: 2024-04-02T05:24:03.283+0000 7fcba0f18700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T05:24:03.583 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:24:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0[95933]: 2024-04-02T05:24:03.283+0000 7fcba0f18700 -1 osd.0 164 *** Got signal Terminated *** 2024-04-02T05:24:03.583 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:24:03 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0[95933]: 2024-04-02T05:24:03.283+0000 7fcba0f18700 -1 osd.0 164 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T05:24:09.021 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:24:08 smithi086 podman[130455]: 2024-04-02 05:24:08.72659786 +0000 UTC m=+5.556847011 container died e521a0ef7dafe5fe9b0fb6ca00f9aadb83ae8f63747c3197f29c0539c46f10a5 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0, io.buildah.version=1.33.5, RELEASE=squid-6cec793, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-04-02T05:24:09.333 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:24:09 smithi086 podman[130455]: 2024-04-02 05:24:09.117942879 +0000 UTC m=+5.948192034 container cleanup e521a0ef7dafe5fe9b0fb6ca00f9aadb83ae8f63747c3197f29c0539c46f10a5 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0, GIT_BRANCH=HEAD, RELEASE=squid-6cec793, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-04-02T05:24:09.333 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:24:09 smithi086 bash[130455]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0 2024-04-02T05:24:09.665 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:24:09 smithi086 podman[130472]: 2024-04-02 05:24:09.344164265 +0000 UTC m=+0.615872582 container remove e521a0ef7dafe5fe9b0fb6ca00f9aadb83ae8f63747c3197f29c0539c46f10a5 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, RELEASE=squid-6cec793) 2024-04-02T05:24:10.033 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:24:09 smithi086 podman[130562]: 2024-04-02 05:24:09.564260205 +0000 UTC m=+0.017929503 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:24:10.033 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:24:09 smithi086 podman[130562]: 2024-04-02 05:24:09.686661789 +0000 UTC m=+0.140331078 container create e93d0c0a48bdcbc6e8b0312b0b89721330bbbc1d70b88076d0e3bd1daa13ffef (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-6cec793, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5) 2024-04-02T05:24:10.033 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:24:09 smithi086 podman[130562]: 2024-04-02 05:24:09.90603778 +0000 UTC m=+0.359707081 container init e93d0c0a48bdcbc6e8b0312b0b89721330bbbc1d70b88076d0e3bd1daa13ffef (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-6cec793, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-04-02T05:24:10.033 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:24:09 smithi086 podman[130562]: 2024-04-02 05:24:09.909522631 +0000 UTC m=+0.363191921 container start e93d0c0a48bdcbc6e8b0312b0b89721330bbbc1d70b88076d0e3bd1daa13ffef (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0-deactivate, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, RELEASE=squid-6cec793, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-02T05:24:10.033 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:24:10 smithi086 podman[130562]: 2024-04-02 05:24:10.031095541 +0000 UTC m=+0.484764849 container attach e93d0c0a48bdcbc6e8b0312b0b89721330bbbc1d70b88076d0e3bd1daa13ffef (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0-deactivate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, GIT_CLEAN=True, RELEASE=squid-6cec793, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-04-02T05:24:10.333 INFO:journalctl@ceph.osd.0.smithi086.stdout:Apr 02 05:24:10 smithi086 podman[130562]: 2024-04-02 05:24:10.200731555 +0000 UTC m=+0.654400837 container died e93d0c0a48bdcbc6e8b0312b0b89721330bbbc1d70b88076d0e3bd1daa13ffef (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-0-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=squid-6cec793, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-04-02T05:24:10.814 DEBUG:teuthology.orchestra.run.smithi086:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.0.service' 2024-04-02T05:24:10.856 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T05:24:10.856 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-04-02T05:24:10.856 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-04-02T05:24:10.857 DEBUG:teuthology.orchestra.run.smithi086:> sudo systemctl stop ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.1 2024-04-02T05:24:11.390 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:24:11 smithi086 systemd[1]: Stopping Ceph osd.1 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:24:11.833 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:24:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1[101169]: 2024-04-02T05:24:11.387+0000 7fbd697bd700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T05:24:11.833 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:24:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1[101169]: 2024-04-02T05:24:11.387+0000 7fbd697bd700 -1 osd.1 164 *** Got signal Terminated *** 2024-04-02T05:24:11.833 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:24:11 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1[101169]: 2024-04-02T05:24:11.387+0000 7fbd697bd700 -1 osd.1 164 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T05:24:17.082 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:24:16 smithi086 podman[130683]: 2024-04-02 05:24:16.810129027 +0000 UTC m=+5.534070506 container died 76fe72844771885bdefd6d204dc4bf7836f688969f0bddcdd30ddf656ce146b7 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=squid-6cec793, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-04-02T05:24:17.432 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:24:17 smithi086 podman[130683]: 2024-04-02 05:24:17.204584392 +0000 UTC m=+5.928525854 container cleanup 76fe72844771885bdefd6d204dc4bf7836f688969f0bddcdd30ddf656ce146b7 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-6cec793, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-04-02T05:24:17.433 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:24:17 smithi086 bash[130683]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1 2024-04-02T05:24:17.747 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:24:17 smithi086 podman[130700]: 2024-04-02 05:24:17.4308155 +0000 UTC m=+0.623308898 container remove 76fe72844771885bdefd6d204dc4bf7836f688969f0bddcdd30ddf656ce146b7 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=squid-6cec793, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-04-02T05:24:18.052 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:24:17 smithi086 podman[130794]: 2024-04-02 05:24:17.647787501 +0000 UTC m=+0.017706622 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:24:18.052 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:24:17 smithi086 podman[130794]: 2024-04-02 05:24:17.756697247 +0000 UTC m=+0.126616357 container create a40c9e165390521681083e40438c750c4337470a3102daa2c7ea12dcad7b53df (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1-deactivate, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=squid-6cec793, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-04-02T05:24:18.053 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:24:17 smithi086 podman[130794]: 2024-04-02 05:24:17.967333851 +0000 UTC m=+0.337252958 container init a40c9e165390521681083e40438c750c4337470a3102daa2c7ea12dcad7b53df (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-6cec793, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True) 2024-04-02T05:24:18.053 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:24:17 smithi086 podman[130794]: 2024-04-02 05:24:17.971052438 +0000 UTC m=+0.340971553 container start a40c9e165390521681083e40438c750c4337470a3102daa2c7ea12dcad7b53df (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1-deactivate, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-6cec793, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-02T05:24:18.053 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:24:18 smithi086 podman[130794]: 2024-04-02 05:24:18.050661755 +0000 UTC m=+0.420580863 container attach a40c9e165390521681083e40438c750c4337470a3102daa2c7ea12dcad7b53df (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1-deactivate, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-6cec793, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-04-02T05:24:18.333 INFO:journalctl@ceph.osd.1.smithi086.stdout:Apr 02 05:24:18 smithi086 podman[130794]: 2024-04-02 05:24:18.26701955 +0000 UTC m=+0.636938649 container died a40c9e165390521681083e40438c750c4337470a3102daa2c7ea12dcad7b53df (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-1-deactivate, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=squid-6cec793, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-04-02T05:24:18.901 DEBUG:teuthology.orchestra.run.smithi086:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.1.service' 2024-04-02T05:24:18.947 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T05:24:18.947 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-04-02T05:24:18.947 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-04-02T05:24:18.947 DEBUG:teuthology.orchestra.run.smithi086:> sudo systemctl stop ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.2 2024-04-02T05:24:19.234 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:24:18 smithi086 systemd[1]: Stopping Ceph osd.2 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:24:19.583 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:24:19 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2[107622]: 2024-04-02T05:24:19.231+0000 7fe7d6129700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T05:24:19.583 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:24:19 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2[107622]: 2024-04-02T05:24:19.231+0000 7fe7d6129700 -1 osd.2 164 *** Got signal Terminated *** 2024-04-02T05:24:19.583 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:24:19 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2[107622]: 2024-04-02T05:24:19.231+0000 7fe7d6129700 -1 osd.2 164 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T05:24:25.027 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:24:24 smithi086 podman[130898]: 2024-04-02 05:24:24.759275774 +0000 UTC m=+5.636929968 container died 28594f089a8f1f8a58edecd0c4cb76aada31a33a43e6c05d3ad84a6d7a852c6b (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=squid-6cec793, ceph=True, org.label-schema.schema-version=1.0) 2024-04-02T05:24:25.333 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:24:25 smithi086 podman[130898]: 2024-04-02 05:24:25.123533082 +0000 UTC m=+6.001187278 container cleanup 28594f089a8f1f8a58edecd0c4cb76aada31a33a43e6c05d3ad84a6d7a852c6b (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, RELEASE=squid-6cec793) 2024-04-02T05:24:25.333 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:24:25 smithi086 bash[130898]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2 2024-04-02T05:24:25.742 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:24:25 smithi086 podman[130917]: 2024-04-02 05:24:25.399915658 +0000 UTC m=+0.636045550 container remove 28594f089a8f1f8a58edecd0c4cb76aada31a33a43e6c05d3ad84a6d7a852c6b (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2, maintainer=Guillaume Abrioux , RELEASE=squid-6cec793, org.label-schema.build-date=20240326, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-02T05:24:26.063 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:24:25 smithi086 podman[131012]: 2024-04-02 05:24:25.64316608 +0000 UTC m=+0.017234081 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:24:26.064 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:24:25 smithi086 podman[131012]: 2024-04-02 05:24:25.759034319 +0000 UTC m=+0.133102353 container create 1cfb58f4a0135ae782c62f1e2293d220fe95bbe4dd2e04a4f8ea76603c572939 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2-deactivate, RELEASE=squid-6cec793, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-02T05:24:26.064 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:24:25 smithi086 podman[131012]: 2024-04-02 05:24:25.978375058 +0000 UTC m=+0.352443061 container init 1cfb58f4a0135ae782c62f1e2293d220fe95bbe4dd2e04a4f8ea76603c572939 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2-deactivate, RELEASE=squid-6cec793, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-04-02T05:24:26.064 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:24:25 smithi086 podman[131012]: 2024-04-02 05:24:25.981632592 +0000 UTC m=+0.355700592 container start 1cfb58f4a0135ae782c62f1e2293d220fe95bbe4dd2e04a4f8ea76603c572939 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2-deactivate, org.label-schema.license=GPLv2, ceph=True, RELEASE=squid-6cec793, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-04-02T05:24:26.064 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:24:26 smithi086 podman[131012]: 2024-04-02 05:24:26.061681254 +0000 UTC m=+0.435749249 container attach 1cfb58f4a0135ae782c62f1e2293d220fe95bbe4dd2e04a4f8ea76603c572939 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=squid-6cec793, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-04-02T05:24:26.333 INFO:journalctl@ceph.osd.2.smithi086.stdout:Apr 02 05:24:26 smithi086 podman[131012]: 2024-04-02 05:24:26.269602265 +0000 UTC m=+0.643670256 container died 1cfb58f4a0135ae782c62f1e2293d220fe95bbe4dd2e04a4f8ea76603c572939 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-2-deactivate, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-6cec793) 2024-04-02T05:24:26.841 DEBUG:teuthology.orchestra.run.smithi086:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.2.service' 2024-04-02T05:24:26.882 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T05:24:26.882 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-04-02T05:24:26.883 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-04-02T05:24:26.883 DEBUG:teuthology.orchestra.run.smithi086:> sudo systemctl stop ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.3 2024-04-02T05:24:27.082 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:24:26 smithi086 systemd[1]: Stopping Ceph osd.3 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:24:27.583 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:24:27 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3[113778]: 2024-04-02T05:24:27.209+0000 7f2e4f872700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T05:24:27.583 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:24:27 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3[113778]: 2024-04-02T05:24:27.209+0000 7f2e4f872700 -1 osd.3 164 *** Got signal Terminated *** 2024-04-02T05:24:27.583 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:24:27 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3[113778]: 2024-04-02T05:24:27.209+0000 7f2e4f872700 -1 osd.3 164 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T05:24:30.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:29 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:29.806+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:07.122343+0000 front 2024-04-02T05:24:07.122198+0000 (oldest deadline 2024-04-02T05:24:28.822138+0000) 2024-04-02T05:24:30.333 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:24:29 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3[113778]: 2024-04-02T05:24:29.894+0000 7f2e4be66700 -1 osd.3 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:08.017105+0000 front 2024-04-02T05:24:08.017251+0000 (oldest deadline 2024-04-02T05:24:29.117284+0000) 2024-04-02T05:24:30.677 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:24:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4[78184]: 2024-04-02T05:24:30.232+0000 7fa4e1a84700 -1 osd.4 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:04.555986+0000 front 2024-04-02T05:24:04.555891+0000 (oldest deadline 2024-04-02T05:24:29.255845+0000) 2024-04-02T05:24:31.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:30.846+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:07.122343+0000 front 2024-04-02T05:24:07.122198+0000 (oldest deadline 2024-04-02T05:24:28.822138+0000) 2024-04-02T05:24:31.177 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:30 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:30.954+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:05.042528+0000 front 2024-04-02T05:24:05.042580+0000 (oldest deadline 2024-04-02T05:24:30.342538+0000) 2024-04-02T05:24:31.332 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:24:30 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3[113778]: 2024-04-02T05:24:30.885+0000 7f2e4be66700 -1 osd.3 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:08.017105+0000 front 2024-04-02T05:24:08.017251+0000 (oldest deadline 2024-04-02T05:24:29.117284+0000) 2024-04-02T05:24:31.677 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:24:31 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4[78184]: 2024-04-02T05:24:31.255+0000 7fa4e1a84700 -1 osd.4 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:04.555986+0000 front 2024-04-02T05:24:04.555891+0000 (oldest deadline 2024-04-02T05:24:29.255845+0000) 2024-04-02T05:24:32.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:31 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:31.822+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:07.122343+0000 front 2024-04-02T05:24:07.122198+0000 (oldest deadline 2024-04-02T05:24:28.822138+0000) 2024-04-02T05:24:32.177 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:31 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:31.956+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:05.042528+0000 front 2024-04-02T05:24:05.042580+0000 (oldest deadline 2024-04-02T05:24:30.342538+0000) 2024-04-02T05:24:32.333 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:24:31 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3[113778]: 2024-04-02T05:24:31.873+0000 7f2e4be66700 -1 osd.3 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:08.017105+0000 front 2024-04-02T05:24:08.017251+0000 (oldest deadline 2024-04-02T05:24:29.117284+0000) 2024-04-02T05:24:32.677 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:24:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4[78184]: 2024-04-02T05:24:32.281+0000 7fa4e1a84700 -1 osd.4 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:04.555986+0000 front 2024-04-02T05:24:04.555891+0000 (oldest deadline 2024-04-02T05:24:29.255845+0000) 2024-04-02T05:24:32.996 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:24:32 smithi086 podman[131128]: 2024-04-02 05:24:32.72827937 +0000 UTC m=+5.624118340 container died 6028f927dfd6758fd491d8de4f84bb5e7fd53eb3a3a4c3a2fe7f464891282bc9 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, RELEASE=squid-6cec793) 2024-04-02T05:24:33.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:32.829+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:07.122343+0000 front 2024-04-02T05:24:07.122198+0000 (oldest deadline 2024-04-02T05:24:28.822138+0000) 2024-04-02T05:24:33.177 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:32 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:32.990+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:05.042528+0000 front 2024-04-02T05:24:05.042580+0000 (oldest deadline 2024-04-02T05:24:30.342538+0000) 2024-04-02T05:24:33.333 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:24:33 smithi086 podman[131128]: 2024-04-02 05:24:33.084777224 +0000 UTC m=+5.980616195 container cleanup 6028f927dfd6758fd491d8de4f84bb5e7fd53eb3a3a4c3a2fe7f464891282bc9 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=squid-6cec793, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, ceph=True) 2024-04-02T05:24:33.333 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:24:33 smithi086 bash[131128]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3 2024-04-02T05:24:33.652 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:24:33 smithi086 podman[131149]: 2024-04-02 05:24:33.35281764 +0000 UTC m=+0.620745049 container remove 6028f927dfd6758fd491d8de4f84bb5e7fd53eb3a3a4c3a2fe7f464891282bc9 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-6cec793, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-04-02T05:24:33.677 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:33 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5[82772]: 2024-04-02T05:24:33.278+0000 7f75c50a5700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:06.995207+0000 front 2024-04-02T05:24:06.995150+0000 (oldest deadline 2024-04-02T05:24:32.295105+0000) 2024-04-02T05:24:33.677 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:24:33 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4[78184]: 2024-04-02T05:24:33.241+0000 7fa4e1a84700 -1 osd.4 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:04.555986+0000 front 2024-04-02T05:24:04.555891+0000 (oldest deadline 2024-04-02T05:24:29.255845+0000) 2024-04-02T05:24:33.949 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:24:33 smithi086 podman[131243]: 2024-04-02 05:24:33.552806162 +0000 UTC m=+0.018207235 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:24:33.950 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:24:33 smithi086 podman[131243]: 2024-04-02 05:24:33.661981058 +0000 UTC m=+0.127382116 container create b77b9b68ca98f2408be189bf6ac3269b7bd0de7285512ddd1c4cb77895091c41 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, RELEASE=squid-6cec793, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-04-02T05:24:33.950 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:24:33 smithi086 podman[131243]: 2024-04-02 05:24:33.864815545 +0000 UTC m=+0.330216610 container init b77b9b68ca98f2408be189bf6ac3269b7bd0de7285512ddd1c4cb77895091c41 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3-deactivate, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=squid-6cec793, org.label-schema.vendor=CentOS) 2024-04-02T05:24:33.950 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:24:33 smithi086 podman[131243]: 2024-04-02 05:24:33.868188008 +0000 UTC m=+0.333589066 container start b77b9b68ca98f2408be189bf6ac3269b7bd0de7285512ddd1c4cb77895091c41 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3-deactivate, io.buildah.version=1.33.5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=squid-6cec793, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD) 2024-04-02T05:24:33.950 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:24:33 smithi086 podman[131243]: 2024-04-02 05:24:33.94765576 +0000 UTC m=+0.413056836 container attach b77b9b68ca98f2408be189bf6ac3269b7bd0de7285512ddd1c4cb77895091c41 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3-deactivate, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=squid-6cec793, CEPH_POINT_RELEASE=) 2024-04-02T05:24:34.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:33 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:33.879+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:07.122343+0000 front 2024-04-02T05:24:07.122198+0000 (oldest deadline 2024-04-02T05:24:28.822138+0000) 2024-04-02T05:24:34.178 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:34 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:34.035+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:05.042528+0000 front 2024-04-02T05:24:05.042580+0000 (oldest deadline 2024-04-02T05:24:30.342538+0000) 2024-04-02T05:24:34.333 INFO:journalctl@ceph.osd.3.smithi086.stdout:Apr 02 05:24:34 smithi086 podman[131243]: 2024-04-02 05:24:34.153628328 +0000 UTC m=+0.619029387 container died b77b9b68ca98f2408be189bf6ac3269b7bd0de7285512ddd1c4cb77895091c41 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, RELEASE=squid-6cec793, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240326) 2024-04-02T05:24:34.677 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:34 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5[82772]: 2024-04-02T05:24:34.286+0000 7f75c50a5700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:06.995207+0000 front 2024-04-02T05:24:06.995150+0000 (oldest deadline 2024-04-02T05:24:32.295105+0000) 2024-04-02T05:24:34.678 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:24:34 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4[78184]: 2024-04-02T05:24:34.231+0000 7fa4e1a84700 -1 osd.4 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:04.555986+0000 front 2024-04-02T05:24:04.555891+0000 (oldest deadline 2024-04-02T05:24:29.255845+0000) 2024-04-02T05:24:34.725 DEBUG:teuthology.orchestra.run.smithi086:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.3.service' 2024-04-02T05:24:34.766 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T05:24:34.766 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-04-02T05:24:34.766 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-04-02T05:24:34.767 DEBUG:teuthology.orchestra.run.smithi192:> sudo systemctl stop ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.4 2024-04-02T05:24:35.060 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:24:34 smithi192 systemd[1]: Stopping Ceph osd.4 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:24:35.061 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:34 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:34.881+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:07.122343+0000 front 2024-04-02T05:24:07.122198+0000 (oldest deadline 2024-04-02T05:24:28.822138+0000) 2024-04-02T05:24:35.061 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:35 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:35.033+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:05.042528+0000 front 2024-04-02T05:24:05.042580+0000 (oldest deadline 2024-04-02T05:24:30.342538+0000) 2024-04-02T05:24:35.427 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:35 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5[82772]: 2024-04-02T05:24:35.280+0000 7f75c50a5700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:06.995207+0000 front 2024-04-02T05:24:06.995150+0000 (oldest deadline 2024-04-02T05:24:32.295105+0000) 2024-04-02T05:24:35.427 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:24:35 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4[78184]: 2024-04-02T05:24:35.057+0000 7fa4e9498700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T05:24:35.427 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:24:35 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4[78184]: 2024-04-02T05:24:35.057+0000 7fa4e9498700 -1 osd.4 164 *** Got signal Terminated *** 2024-04-02T05:24:35.427 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:24:35 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4[78184]: 2024-04-02T05:24:35.057+0000 7fa4e9498700 -1 osd.4 164 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T05:24:35.428 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:24:35 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4[78184]: 2024-04-02T05:24:35.224+0000 7fa4e1a84700 -1 osd.4 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:04.555986+0000 front 2024-04-02T05:24:04.555891+0000 (oldest deadline 2024-04-02T05:24:29.255845+0000) 2024-04-02T05:24:36.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:35 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:35.842+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:07.122343+0000 front 2024-04-02T05:24:07.122198+0000 (oldest deadline 2024-04-02T05:24:28.822138+0000) 2024-04-02T05:24:36.177 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:35 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:35.996+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:05.042528+0000 front 2024-04-02T05:24:05.042580+0000 (oldest deadline 2024-04-02T05:24:30.342538+0000) 2024-04-02T05:24:36.677 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:24:36 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4[78184]: 2024-04-02T05:24:36.226+0000 7fa4e1a84700 -1 osd.4 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:04.555986+0000 front 2024-04-02T05:24:04.555891+0000 (oldest deadline 2024-04-02T05:24:29.255845+0000) 2024-04-02T05:24:36.677 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:36 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5[82772]: 2024-04-02T05:24:36.310+0000 7f75c50a5700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:06.995207+0000 front 2024-04-02T05:24:06.995150+0000 (oldest deadline 2024-04-02T05:24:32.295105+0000) 2024-04-02T05:24:37.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:36 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:36.825+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:07.122343+0000 front 2024-04-02T05:24:07.122198+0000 (oldest deadline 2024-04-02T05:24:28.822138+0000) 2024-04-02T05:24:37.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:36 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:36.825+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.724076+0000 front 2024-04-02T05:24:15.724088+0000 (oldest deadline 2024-04-02T05:24:36.823875+0000) 2024-04-02T05:24:37.178 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:36 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:36.967+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:05.042528+0000 front 2024-04-02T05:24:05.042580+0000 (oldest deadline 2024-04-02T05:24:30.342538+0000) 2024-04-02T05:24:37.492 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:24:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4[78184]: 2024-04-02T05:24:37.238+0000 7fa4e1a84700 -1 osd.4 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:04.555986+0000 front 2024-04-02T05:24:04.555891+0000 (oldest deadline 2024-04-02T05:24:29.255845+0000) 2024-04-02T05:24:37.493 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5[82772]: 2024-04-02T05:24:37.337+0000 7f75c50a5700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:06.995207+0000 front 2024-04-02T05:24:06.995150+0000 (oldest deadline 2024-04-02T05:24:32.295105+0000) 2024-04-02T05:24:37.493 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:24:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:24:37.489Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.86:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.86:8765: connect: connection refused" 2024-04-02T05:24:37.818 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:24:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:24:37.490Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.86:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.86:8765: connect: connection refused" 2024-04-02T05:24:37.818 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:24:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:24:37.491Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.86:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.86:8765: connect: connection refused" 2024-04-02T05:24:37.818 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:24:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:24:37.491Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.86:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.86:8765: connect: connection refused" 2024-04-02T05:24:38.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:37.815+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:07.122343+0000 front 2024-04-02T05:24:07.122198+0000 (oldest deadline 2024-04-02T05:24:28.822138+0000) 2024-04-02T05:24:38.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:37.815+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.724076+0000 front 2024-04-02T05:24:15.724088+0000 (oldest deadline 2024-04-02T05:24:36.823875+0000) 2024-04-02T05:24:38.178 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:37.997+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:05.042528+0000 front 2024-04-02T05:24:05.042580+0000 (oldest deadline 2024-04-02T05:24:30.342538+0000) 2024-04-02T05:24:38.178 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:37.997+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.043628+0000 front 2024-04-02T05:24:15.043547+0000 (oldest deadline 2024-04-02T05:24:37.943436+0000) 2024-04-02T05:24:38.677 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:24:38 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4[78184]: 2024-04-02T05:24:38.273+0000 7fa4e1a84700 -1 osd.4 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:04.555986+0000 front 2024-04-02T05:24:04.555891+0000 (oldest deadline 2024-04-02T05:24:29.255845+0000) 2024-04-02T05:24:38.677 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:38 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5[82772]: 2024-04-02T05:24:38.290+0000 7f75c50a5700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:06.995207+0000 front 2024-04-02T05:24:06.995150+0000 (oldest deadline 2024-04-02T05:24:32.295105+0000) 2024-04-02T05:24:39.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:38 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:38.791+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:07.122343+0000 front 2024-04-02T05:24:07.122198+0000 (oldest deadline 2024-04-02T05:24:28.822138+0000) 2024-04-02T05:24:39.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:38 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:38.791+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.724076+0000 front 2024-04-02T05:24:15.724088+0000 (oldest deadline 2024-04-02T05:24:36.823875+0000) 2024-04-02T05:24:39.178 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:38 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:38.993+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:05.042528+0000 front 2024-04-02T05:24:05.042580+0000 (oldest deadline 2024-04-02T05:24:30.342538+0000) 2024-04-02T05:24:39.178 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:38 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:38.993+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.043628+0000 front 2024-04-02T05:24:15.043547+0000 (oldest deadline 2024-04-02T05:24:37.943436+0000) 2024-04-02T05:24:39.677 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:24:39 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4[78184]: 2024-04-02T05:24:39.237+0000 7fa4e1a84700 -1 osd.4 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:04.555986+0000 front 2024-04-02T05:24:04.555891+0000 (oldest deadline 2024-04-02T05:24:29.255845+0000) 2024-04-02T05:24:39.678 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:24:39 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4[78184]: 2024-04-02T05:24:39.237+0000 7fa4e1a84700 -1 osd.4 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.657782+0000 front 2024-04-02T05:24:15.657724+0000 (oldest deadline 2024-04-02T05:24:39.157666+0000) 2024-04-02T05:24:39.678 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:39 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5[82772]: 2024-04-02T05:24:39.288+0000 7f75c50a5700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:06.995207+0000 front 2024-04-02T05:24:06.995150+0000 (oldest deadline 2024-04-02T05:24:32.295105+0000) 2024-04-02T05:24:39.678 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:39 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5[82772]: 2024-04-02T05:24:39.288+0000 7f75c50a5700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.696490+0000 front 2024-04-02T05:24:15.696399+0000 (oldest deadline 2024-04-02T05:24:38.596338+0000) 2024-04-02T05:24:40.178 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:39 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:39.809+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:07.122343+0000 front 2024-04-02T05:24:07.122198+0000 (oldest deadline 2024-04-02T05:24:28.822138+0000) 2024-04-02T05:24:40.178 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:39 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:39.809+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.724076+0000 front 2024-04-02T05:24:15.724088+0000 (oldest deadline 2024-04-02T05:24:36.823875+0000) 2024-04-02T05:24:40.178 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:40 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:40.026+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:05.042528+0000 front 2024-04-02T05:24:05.042580+0000 (oldest deadline 2024-04-02T05:24:30.342538+0000) 2024-04-02T05:24:40.178 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:40 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:40.026+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.043628+0000 front 2024-04-02T05:24:15.043547+0000 (oldest deadline 2024-04-02T05:24:37.943436+0000) 2024-04-02T05:24:40.601 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:40 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5[82772]: 2024-04-02T05:24:40.305+0000 7f75c50a5700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:06.995207+0000 front 2024-04-02T05:24:06.995150+0000 (oldest deadline 2024-04-02T05:24:32.295105+0000) 2024-04-02T05:24:40.601 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:40 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5[82772]: 2024-04-02T05:24:40.305+0000 7f75c50a5700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.696490+0000 front 2024-04-02T05:24:15.696399+0000 (oldest deadline 2024-04-02T05:24:38.596338+0000) 2024-04-02T05:24:40.852 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:24:40 smithi192 podman[101553]: 2024-04-02 05:24:40.599245214 +0000 UTC m=+5.640062170 container died f9c58b761b6244509db53e2be5a3b69be2053c6c5b963a20e44773fb123771c0 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=squid-6cec793, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-02T05:24:41.177 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:24:41 smithi192 podman[101553]: 2024-04-02 05:24:41.000377534 +0000 UTC m=+6.041194501 container cleanup f9c58b761b6244509db53e2be5a3b69be2053c6c5b963a20e44773fb123771c0 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4, RELEASE=squid-6cec793, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0) 2024-04-02T05:24:41.178 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:24:41 smithi192 bash[101553]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4 2024-04-02T05:24:41.178 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:40 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:40.849+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:07.122343+0000 front 2024-04-02T05:24:07.122198+0000 (oldest deadline 2024-04-02T05:24:28.822138+0000) 2024-04-02T05:24:41.178 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:40 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:40.849+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.724076+0000 front 2024-04-02T05:24:15.724088+0000 (oldest deadline 2024-04-02T05:24:36.823875+0000) 2024-04-02T05:24:41.178 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:41 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:41.067+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:05.042528+0000 front 2024-04-02T05:24:05.042580+0000 (oldest deadline 2024-04-02T05:24:30.342538+0000) 2024-04-02T05:24:41.178 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:41 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:41.067+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.043628+0000 front 2024-04-02T05:24:15.043547+0000 (oldest deadline 2024-04-02T05:24:37.943436+0000) 2024-04-02T05:24:41.579 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:24:41 smithi192 podman[101570]: 2024-04-02 05:24:41.285914375 +0000 UTC m=+0.682494386 container remove f9c58b761b6244509db53e2be5a3b69be2053c6c5b963a20e44773fb123771c0 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=squid-6cec793, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0) 2024-04-02T05:24:41.579 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:41 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5[82772]: 2024-04-02T05:24:41.302+0000 7f75c50a5700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:06.995207+0000 front 2024-04-02T05:24:06.995150+0000 (oldest deadline 2024-04-02T05:24:32.295105+0000) 2024-04-02T05:24:41.580 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:41 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5[82772]: 2024-04-02T05:24:41.302+0000 7f75c50a5700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.696490+0000 front 2024-04-02T05:24:15.696399+0000 (oldest deadline 2024-04-02T05:24:38.596338+0000) 2024-04-02T05:24:41.834 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:41 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:41.804+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:07.122343+0000 front 2024-04-02T05:24:07.122198+0000 (oldest deadline 2024-04-02T05:24:28.822138+0000) 2024-04-02T05:24:41.834 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:41 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:41.804+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.724076+0000 front 2024-04-02T05:24:15.724088+0000 (oldest deadline 2024-04-02T05:24:36.823875+0000) 2024-04-02T05:24:41.835 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:24:41 smithi192 podman[101661]: 2024-04-02 05:24:41.480088417 +0000 UTC m=+0.018520015 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:24:41.835 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:24:41 smithi192 podman[101661]: 2024-04-02 05:24:41.604241093 +0000 UTC m=+0.142672678 container create b302f8a2b7a4c731566d142151417244bd8ecf68805824f30383eab4b77107c6 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, CEPH_POINT_RELEASE=, RELEASE=squid-6cec793, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-04-02T05:24:42.114 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:24:41 smithi192 podman[101661]: 2024-04-02 05:24:41.832426337 +0000 UTC m=+0.370857923 container init b302f8a2b7a4c731566d142151417244bd8ecf68805824f30383eab4b77107c6 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=squid-6cec793, maintainer=Guillaume Abrioux ) 2024-04-02T05:24:42.114 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:24:41 smithi192 podman[101661]: 2024-04-02 05:24:41.836108398 +0000 UTC m=+0.374539987 container start b302f8a2b7a4c731566d142151417244bd8ecf68805824f30383eab4b77107c6 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=squid-6cec793, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-04-02T05:24:42.114 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:24:41 smithi192 podman[101661]: 2024-04-02 05:24:41.932114313 +0000 UTC m=+0.470545913 container attach b302f8a2b7a4c731566d142151417244bd8ecf68805824f30383eab4b77107c6 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4-deactivate, ceph=True, org.label-schema.schema-version=1.0, RELEASE=squid-6cec793, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-04-02T05:24:42.384 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:42 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:42.111+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:05.042528+0000 front 2024-04-02T05:24:05.042580+0000 (oldest deadline 2024-04-02T05:24:30.342538+0000) 2024-04-02T05:24:42.384 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:42 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:42.111+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.043628+0000 front 2024-04-02T05:24:15.043547+0000 (oldest deadline 2024-04-02T05:24:37.943436+0000) 2024-04-02T05:24:42.384 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:42 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5[82772]: 2024-04-02T05:24:42.261+0000 7f75c50a5700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:06.995207+0000 front 2024-04-02T05:24:06.995150+0000 (oldest deadline 2024-04-02T05:24:32.295105+0000) 2024-04-02T05:24:42.385 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:42 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5[82772]: 2024-04-02T05:24:42.261+0000 7f75c50a5700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.696490+0000 front 2024-04-02T05:24:15.696399+0000 (oldest deadline 2024-04-02T05:24:38.596338+0000) 2024-04-02T05:24:42.385 INFO:journalctl@ceph.osd.4.smithi192.stdout:Apr 02 05:24:42 smithi192 podman[101661]: 2024-04-02 05:24:42.126972367 +0000 UTC m=+0.665403955 container died b302f8a2b7a4c731566d142151417244bd8ecf68805824f30383eab4b77107c6 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-4-deactivate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-6cec793, CEPH_POINT_RELEASE=) 2024-04-02T05:24:42.744 DEBUG:teuthology.orchestra.run.smithi192:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.4.service' 2024-04-02T05:24:42.785 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T05:24:42.785 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-04-02T05:24:42.786 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-04-02T05:24:42.786 DEBUG:teuthology.orchestra.run.smithi192:> sudo systemctl stop ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.5 2024-04-02T05:24:43.116 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:42 smithi192 systemd[1]: Stopping Ceph osd.5 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:24:43.116 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:42 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:42.778+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:07.122343+0000 front 2024-04-02T05:24:07.122198+0000 (oldest deadline 2024-04-02T05:24:28.822138+0000) 2024-04-02T05:24:43.116 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:42 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:42.778+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.724076+0000 front 2024-04-02T05:24:15.724088+0000 (oldest deadline 2024-04-02T05:24:36.823875+0000) 2024-04-02T05:24:43.427 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:43 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5[82772]: 2024-04-02T05:24:43.155+0000 7f75ccab9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T05:24:43.428 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:43 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5[82772]: 2024-04-02T05:24:43.155+0000 7f75ccab9700 -1 osd.5 164 *** Got signal Terminated *** 2024-04-02T05:24:43.428 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:43 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5[82772]: 2024-04-02T05:24:43.155+0000 7f75ccab9700 -1 osd.5 164 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T05:24:43.428 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:43 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5[82772]: 2024-04-02T05:24:43.240+0000 7f75c50a5700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:06.995207+0000 front 2024-04-02T05:24:06.995150+0000 (oldest deadline 2024-04-02T05:24:32.295105+0000) 2024-04-02T05:24:43.428 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:43 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5[82772]: 2024-04-02T05:24:43.240+0000 7f75c50a5700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.696490+0000 front 2024-04-02T05:24:15.696399+0000 (oldest deadline 2024-04-02T05:24:38.596338+0000) 2024-04-02T05:24:43.428 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:43 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:43.113+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:05.042528+0000 front 2024-04-02T05:24:05.042580+0000 (oldest deadline 2024-04-02T05:24:30.342538+0000) 2024-04-02T05:24:43.428 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:43 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:43.113+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.043628+0000 front 2024-04-02T05:24:15.043547+0000 (oldest deadline 2024-04-02T05:24:37.943436+0000) 2024-04-02T05:24:44.142 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:43 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:43.791+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:07.122343+0000 front 2024-04-02T05:24:07.122198+0000 (oldest deadline 2024-04-02T05:24:28.822138+0000) 2024-04-02T05:24:44.142 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:43 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:43.791+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.724076+0000 front 2024-04-02T05:24:15.724088+0000 (oldest deadline 2024-04-02T05:24:36.823875+0000) 2024-04-02T05:24:44.427 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:44 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5[82772]: 2024-04-02T05:24:44.239+0000 7f75c50a5700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:06.995207+0000 front 2024-04-02T05:24:06.995150+0000 (oldest deadline 2024-04-02T05:24:32.295105+0000) 2024-04-02T05:24:44.427 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:44 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5[82772]: 2024-04-02T05:24:44.239+0000 7f75c50a5700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.696490+0000 front 2024-04-02T05:24:15.696399+0000 (oldest deadline 2024-04-02T05:24:38.596338+0000) 2024-04-02T05:24:44.428 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:44 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:44.139+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:05.042528+0000 front 2024-04-02T05:24:05.042580+0000 (oldest deadline 2024-04-02T05:24:30.342538+0000) 2024-04-02T05:24:44.428 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:44 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:44.139+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.043628+0000 front 2024-04-02T05:24:15.043547+0000 (oldest deadline 2024-04-02T05:24:37.943436+0000) 2024-04-02T05:24:45.165 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:44 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:44.779+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:07.122343+0000 front 2024-04-02T05:24:07.122198+0000 (oldest deadline 2024-04-02T05:24:28.822138+0000) 2024-04-02T05:24:45.165 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:44 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:44.779+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.724076+0000 front 2024-04-02T05:24:15.724088+0000 (oldest deadline 2024-04-02T05:24:36.823875+0000) 2024-04-02T05:24:45.427 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:45 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5[82772]: 2024-04-02T05:24:45.274+0000 7f75c50a5700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:06.995207+0000 front 2024-04-02T05:24:06.995150+0000 (oldest deadline 2024-04-02T05:24:32.295105+0000) 2024-04-02T05:24:45.427 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:45 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5[82772]: 2024-04-02T05:24:45.275+0000 7f75c50a5700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.696490+0000 front 2024-04-02T05:24:15.696399+0000 (oldest deadline 2024-04-02T05:24:38.596338+0000) 2024-04-02T05:24:45.428 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:45 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:45.162+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:05.042528+0000 front 2024-04-02T05:24:05.042580+0000 (oldest deadline 2024-04-02T05:24:30.342538+0000) 2024-04-02T05:24:45.428 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:45 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:45.162+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.043628+0000 front 2024-04-02T05:24:15.043547+0000 (oldest deadline 2024-04-02T05:24:37.943436+0000) 2024-04-02T05:24:46.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:45 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:45.794+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:07.122343+0000 front 2024-04-02T05:24:07.122198+0000 (oldest deadline 2024-04-02T05:24:28.822138+0000) 2024-04-02T05:24:46.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:45 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:45.794+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.724076+0000 front 2024-04-02T05:24:15.724088+0000 (oldest deadline 2024-04-02T05:24:36.823875+0000) 2024-04-02T05:24:46.677 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:46 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5[82772]: 2024-04-02T05:24:46.298+0000 7f75c50a5700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:06.995207+0000 front 2024-04-02T05:24:06.995150+0000 (oldest deadline 2024-04-02T05:24:32.295105+0000) 2024-04-02T05:24:46.678 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:46 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5[82772]: 2024-04-02T05:24:46.298+0000 7f75c50a5700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.696490+0000 front 2024-04-02T05:24:15.696399+0000 (oldest deadline 2024-04-02T05:24:38.596338+0000) 2024-04-02T05:24:46.678 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:46 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:46.206+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:05.042528+0000 front 2024-04-02T05:24:05.042580+0000 (oldest deadline 2024-04-02T05:24:30.342538+0000) 2024-04-02T05:24:46.678 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:46 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:46.206+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.043628+0000 front 2024-04-02T05:24:15.043547+0000 (oldest deadline 2024-04-02T05:24:37.943436+0000) 2024-04-02T05:24:46.678 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:46 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:46.206+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6822 osd.2 since back 2024-04-02T05:24:20.244434+0000 front 2024-04-02T05:24:20.244508+0000 (oldest deadline 2024-04-02T05:24:45.544290+0000) 2024-04-02T05:24:47.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:46 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:46.827+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:07.122343+0000 front 2024-04-02T05:24:07.122198+0000 (oldest deadline 2024-04-02T05:24:28.822138+0000) 2024-04-02T05:24:47.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:46 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:46.827+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.724076+0000 front 2024-04-02T05:24:15.724088+0000 (oldest deadline 2024-04-02T05:24:36.823875+0000) 2024-04-02T05:24:47.677 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5[82772]: 2024-04-02T05:24:47.303+0000 7f75c50a5700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:06.995207+0000 front 2024-04-02T05:24:06.995150+0000 (oldest deadline 2024-04-02T05:24:32.295105+0000) 2024-04-02T05:24:47.677 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5[82772]: 2024-04-02T05:24:47.303+0000 7f75c50a5700 -1 osd.5 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.696490+0000 front 2024-04-02T05:24:15.696399+0000 (oldest deadline 2024-04-02T05:24:38.596338+0000) 2024-04-02T05:24:47.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:47.213+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:05.042528+0000 front 2024-04-02T05:24:05.042580+0000 (oldest deadline 2024-04-02T05:24:30.342538+0000) 2024-04-02T05:24:47.678 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:47.213+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.043628+0000 front 2024-04-02T05:24:15.043547+0000 (oldest deadline 2024-04-02T05:24:37.943436+0000) 2024-04-02T05:24:47.678 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:47.213+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6822 osd.2 since back 2024-04-02T05:24:20.244434+0000 front 2024-04-02T05:24:20.244508+0000 (oldest deadline 2024-04-02T05:24:45.544290+0000) 2024-04-02T05:24:48.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:47.861+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:07.122343+0000 front 2024-04-02T05:24:07.122198+0000 (oldest deadline 2024-04-02T05:24:28.822138+0000) 2024-04-02T05:24:48.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:47 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:47.861+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.724076+0000 front 2024-04-02T05:24:15.724088+0000 (oldest deadline 2024-04-02T05:24:36.823875+0000) 2024-04-02T05:24:48.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:48 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:48.211+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:05.042528+0000 front 2024-04-02T05:24:05.042580+0000 (oldest deadline 2024-04-02T05:24:30.342538+0000) 2024-04-02T05:24:48.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:48 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:48.211+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.043628+0000 front 2024-04-02T05:24:15.043547+0000 (oldest deadline 2024-04-02T05:24:37.943436+0000) 2024-04-02T05:24:48.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:48 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:48.211+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6822 osd.2 since back 2024-04-02T05:24:20.244434+0000 front 2024-04-02T05:24:20.244508+0000 (oldest deadline 2024-04-02T05:24:45.544290+0000) 2024-04-02T05:24:48.951 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:48 smithi192 podman[101778]: 2024-04-02 05:24:48.689139363 +0000 UTC m=+5.664214458 container died 813afd5cf75d58344b1b028e854689b6efdddb8eea18970acf09c34dc325d1e2 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5, ceph=True, org.label-schema.license=GPLv2, RELEASE=squid-6cec793, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS) 2024-04-02T05:24:48.951 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:48 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:48.829+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:07.122343+0000 front 2024-04-02T05:24:07.122198+0000 (oldest deadline 2024-04-02T05:24:28.822138+0000) 2024-04-02T05:24:48.952 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:48 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:48.829+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.724076+0000 front 2024-04-02T05:24:15.724088+0000 (oldest deadline 2024-04-02T05:24:36.823875+0000) 2024-04-02T05:24:48.952 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:48 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:48.829+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6822 osd.2 since back 2024-04-02T05:24:23.825212+0000 front 2024-04-02T05:24:23.825300+0000 (oldest deadline 2024-04-02T05:24:48.525094+0000) 2024-04-02T05:24:49.221 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:49 smithi192 podman[101778]: 2024-04-02 05:24:49.114839667 +0000 UTC m=+6.089914802 container cleanup 813afd5cf75d58344b1b028e854689b6efdddb8eea18970acf09c34dc325d1e2 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=squid-6cec793, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-02T05:24:49.221 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:49 smithi192 bash[101778]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5 2024-04-02T05:24:49.677 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:49 smithi192 podman[101796]: 2024-04-02 05:24:49.316368971 +0000 UTC m=+0.625935134 container remove 813afd5cf75d58344b1b028e854689b6efdddb8eea18970acf09c34dc325d1e2 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-6cec793, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-04-02T05:24:49.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:49 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:49.218+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:05.042528+0000 front 2024-04-02T05:24:05.042580+0000 (oldest deadline 2024-04-02T05:24:30.342538+0000) 2024-04-02T05:24:49.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:49 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:49.218+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.043628+0000 front 2024-04-02T05:24:15.043547+0000 (oldest deadline 2024-04-02T05:24:37.943436+0000) 2024-04-02T05:24:49.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:49 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:49.218+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6822 osd.2 since back 2024-04-02T05:24:20.244434+0000 front 2024-04-02T05:24:20.244508+0000 (oldest deadline 2024-04-02T05:24:45.544290+0000) 2024-04-02T05:24:50.013 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:49 smithi192 podman[101889]: 2024-04-02 05:24:49.629389194 +0000 UTC m=+0.017756544 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:24:50.014 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:49 smithi192 podman[101889]: 2024-04-02 05:24:49.733951612 +0000 UTC m=+0.122318947 container create a7b2822a7692152fad15e6cb47194d75b1ab2d84e457a00e315dca73e59aeaba (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, RELEASE=squid-6cec793, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-02T05:24:50.014 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:49 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:49.791+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:07.122343+0000 front 2024-04-02T05:24:07.122198+0000 (oldest deadline 2024-04-02T05:24:28.822138+0000) 2024-04-02T05:24:50.014 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:49 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:49.791+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.724076+0000 front 2024-04-02T05:24:15.724088+0000 (oldest deadline 2024-04-02T05:24:36.823875+0000) 2024-04-02T05:24:50.014 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:49 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:49.791+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6822 osd.2 since back 2024-04-02T05:24:23.825212+0000 front 2024-04-02T05:24:23.825300+0000 (oldest deadline 2024-04-02T05:24:48.525094+0000) 2024-04-02T05:24:50.306 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:50 smithi192 podman[101889]: 2024-04-02 05:24:50.011574826 +0000 UTC m=+0.399942158 container init a7b2822a7692152fad15e6cb47194d75b1ab2d84e457a00e315dca73e59aeaba (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=squid-6cec793, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240326, GIT_CLEAN=True) 2024-04-02T05:24:50.306 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:50 smithi192 podman[101889]: 2024-04-02 05:24:50.014861472 +0000 UTC m=+0.403228810 container start a7b2822a7692152fad15e6cb47194d75b1ab2d84e457a00e315dca73e59aeaba (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=squid-6cec793, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0) 2024-04-02T05:24:50.306 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:50 smithi192 podman[101889]: 2024-04-02 05:24:50.111649802 +0000 UTC m=+0.500017143 container attach a7b2822a7692152fad15e6cb47194d75b1ab2d84e457a00e315dca73e59aeaba (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5-deactivate, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-6cec793, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326) 2024-04-02T05:24:50.306 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:50 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:50.194+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:05.042528+0000 front 2024-04-02T05:24:05.042580+0000 (oldest deadline 2024-04-02T05:24:30.342538+0000) 2024-04-02T05:24:50.307 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:50 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:50.194+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.043628+0000 front 2024-04-02T05:24:15.043547+0000 (oldest deadline 2024-04-02T05:24:37.943436+0000) 2024-04-02T05:24:50.307 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:50 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:50.194+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6822 osd.2 since back 2024-04-02T05:24:20.244434+0000 front 2024-04-02T05:24:20.244508+0000 (oldest deadline 2024-04-02T05:24:45.544290+0000) 2024-04-02T05:24:50.594 INFO:journalctl@ceph.osd.5.smithi192.stdout:Apr 02 05:24:50 smithi192 podman[101889]: 2024-04-02 05:24:50.304913969 +0000 UTC m=+0.693281328 container died a7b2822a7692152fad15e6cb47194d75b1ab2d84e457a00e315dca73e59aeaba (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-5-deactivate, RELEASE=squid-6cec793, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-02T05:24:50.884 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:50 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:50.770+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:07.122343+0000 front 2024-04-02T05:24:07.122198+0000 (oldest deadline 2024-04-02T05:24:28.822138+0000) 2024-04-02T05:24:50.884 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:50 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:50.770+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.724076+0000 front 2024-04-02T05:24:15.724088+0000 (oldest deadline 2024-04-02T05:24:36.823875+0000) 2024-04-02T05:24:50.884 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:50 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:50.770+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6822 osd.2 since back 2024-04-02T05:24:23.825212+0000 front 2024-04-02T05:24:23.825300+0000 (oldest deadline 2024-04-02T05:24:48.525094+0000) 2024-04-02T05:24:50.900 DEBUG:teuthology.orchestra.run.smithi192:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.5.service' 2024-04-02T05:24:50.944 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T05:24:50.944 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-04-02T05:24:50.945 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-04-02T05:24:50.945 DEBUG:teuthology.orchestra.run.smithi192:> sudo systemctl stop ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.6 2024-04-02T05:24:51.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:51 smithi192 systemd[1]: Stopping Ceph osd.6 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:24:51.677 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:51 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:51.251+0000 7f97224bb700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T05:24:51.678 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:51 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:51.251+0000 7f97224bb700 -1 osd.6 164 *** Got signal Terminated *** 2024-04-02T05:24:51.678 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:51 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:51.251+0000 7f97224bb700 -1 osd.6 164 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T05:24:51.678 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:51 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:51.212+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:05.042528+0000 front 2024-04-02T05:24:05.042580+0000 (oldest deadline 2024-04-02T05:24:30.342538+0000) 2024-04-02T05:24:51.678 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:51 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:51.212+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.043628+0000 front 2024-04-02T05:24:15.043547+0000 (oldest deadline 2024-04-02T05:24:37.943436+0000) 2024-04-02T05:24:51.678 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:51 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:51.212+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6822 osd.2 since back 2024-04-02T05:24:20.244434+0000 front 2024-04-02T05:24:20.244508+0000 (oldest deadline 2024-04-02T05:24:45.544290+0000) 2024-04-02T05:24:52.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:51 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:51.814+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:07.122343+0000 front 2024-04-02T05:24:07.122198+0000 (oldest deadline 2024-04-02T05:24:28.822138+0000) 2024-04-02T05:24:52.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:51 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:51.814+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.724076+0000 front 2024-04-02T05:24:15.724088+0000 (oldest deadline 2024-04-02T05:24:36.823875+0000) 2024-04-02T05:24:52.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:51 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:51.814+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6822 osd.2 since back 2024-04-02T05:24:23.825212+0000 front 2024-04-02T05:24:23.825300+0000 (oldest deadline 2024-04-02T05:24:48.525094+0000) 2024-04-02T05:24:52.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:52 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:52.230+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:05.042528+0000 front 2024-04-02T05:24:05.042580+0000 (oldest deadline 2024-04-02T05:24:30.342538+0000) 2024-04-02T05:24:52.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:52 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:52.230+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.043628+0000 front 2024-04-02T05:24:15.043547+0000 (oldest deadline 2024-04-02T05:24:37.943436+0000) 2024-04-02T05:24:52.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:52 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:52.230+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6822 osd.2 since back 2024-04-02T05:24:20.244434+0000 front 2024-04-02T05:24:20.244508+0000 (oldest deadline 2024-04-02T05:24:45.544290+0000) 2024-04-02T05:24:53.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:52 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:52.795+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:07.122343+0000 front 2024-04-02T05:24:07.122198+0000 (oldest deadline 2024-04-02T05:24:28.822138+0000) 2024-04-02T05:24:53.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:52 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:52.795+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.724076+0000 front 2024-04-02T05:24:15.724088+0000 (oldest deadline 2024-04-02T05:24:36.823875+0000) 2024-04-02T05:24:53.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:52 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:52.795+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6822 osd.2 since back 2024-04-02T05:24:23.825212+0000 front 2024-04-02T05:24:23.825300+0000 (oldest deadline 2024-04-02T05:24:48.525094+0000) 2024-04-02T05:24:53.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:53 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:53.257+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:05.042528+0000 front 2024-04-02T05:24:05.042580+0000 (oldest deadline 2024-04-02T05:24:30.342538+0000) 2024-04-02T05:24:53.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:53 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:53.257+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.043628+0000 front 2024-04-02T05:24:15.043547+0000 (oldest deadline 2024-04-02T05:24:37.943436+0000) 2024-04-02T05:24:53.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:53 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:53.257+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6822 osd.2 since back 2024-04-02T05:24:20.244434+0000 front 2024-04-02T05:24:20.244508+0000 (oldest deadline 2024-04-02T05:24:45.544290+0000) 2024-04-02T05:24:54.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:53 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:53.828+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:07.122343+0000 front 2024-04-02T05:24:07.122198+0000 (oldest deadline 2024-04-02T05:24:28.822138+0000) 2024-04-02T05:24:54.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:53 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:53.828+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.724076+0000 front 2024-04-02T05:24:15.724088+0000 (oldest deadline 2024-04-02T05:24:36.823875+0000) 2024-04-02T05:24:54.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:53 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:53.828+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6822 osd.2 since back 2024-04-02T05:24:23.825212+0000 front 2024-04-02T05:24:23.825300+0000 (oldest deadline 2024-04-02T05:24:48.525094+0000) 2024-04-02T05:24:54.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:54 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:54.220+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:05.042528+0000 front 2024-04-02T05:24:05.042580+0000 (oldest deadline 2024-04-02T05:24:30.342538+0000) 2024-04-02T05:24:54.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:54 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:54.220+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.043628+0000 front 2024-04-02T05:24:15.043547+0000 (oldest deadline 2024-04-02T05:24:37.943436+0000) 2024-04-02T05:24:54.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:54 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:54.220+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6822 osd.2 since back 2024-04-02T05:24:20.244434+0000 front 2024-04-02T05:24:20.244508+0000 (oldest deadline 2024-04-02T05:24:45.544290+0000) 2024-04-02T05:24:54.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:54 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:54.220+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6830 osd.3 since back 2024-04-02T05:24:30.845331+0000 front 2024-04-02T05:24:30.845284+0000 (oldest deadline 2024-04-02T05:24:53.745178+0000) 2024-04-02T05:24:55.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:54 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:54.804+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:07.122343+0000 front 2024-04-02T05:24:07.122198+0000 (oldest deadline 2024-04-02T05:24:28.822138+0000) 2024-04-02T05:24:55.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:54 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:54.804+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.724076+0000 front 2024-04-02T05:24:15.724088+0000 (oldest deadline 2024-04-02T05:24:36.823875+0000) 2024-04-02T05:24:55.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:54 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:54.804+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6822 osd.2 since back 2024-04-02T05:24:23.825212+0000 front 2024-04-02T05:24:23.825300+0000 (oldest deadline 2024-04-02T05:24:48.525094+0000) 2024-04-02T05:24:55.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:55 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:55.228+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:05.042528+0000 front 2024-04-02T05:24:05.042580+0000 (oldest deadline 2024-04-02T05:24:30.342538+0000) 2024-04-02T05:24:55.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:55 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:55.228+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.043628+0000 front 2024-04-02T05:24:15.043547+0000 (oldest deadline 2024-04-02T05:24:37.943436+0000) 2024-04-02T05:24:55.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:55 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:55.228+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6822 osd.2 since back 2024-04-02T05:24:20.244434+0000 front 2024-04-02T05:24:20.244508+0000 (oldest deadline 2024-04-02T05:24:45.544290+0000) 2024-04-02T05:24:55.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:55 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:55.228+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6830 osd.3 since back 2024-04-02T05:24:30.845331+0000 front 2024-04-02T05:24:30.845284+0000 (oldest deadline 2024-04-02T05:24:53.745178+0000) 2024-04-02T05:24:56.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:55 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:55.847+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:07.122343+0000 front 2024-04-02T05:24:07.122198+0000 (oldest deadline 2024-04-02T05:24:28.822138+0000) 2024-04-02T05:24:56.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:55 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:55.847+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.724076+0000 front 2024-04-02T05:24:15.724088+0000 (oldest deadline 2024-04-02T05:24:36.823875+0000) 2024-04-02T05:24:56.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:55 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6[87353]: 2024-04-02T05:24:55.847+0000 7f971aaa7700 -1 osd.6 164 heartbeat_check: no reply from 172.21.15.86:6822 osd.2 since back 2024-04-02T05:24:23.825212+0000 front 2024-04-02T05:24:23.825300+0000 (oldest deadline 2024-04-02T05:24:48.525094+0000) 2024-04-02T05:24:56.657 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:56 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:56.265+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:05.042528+0000 front 2024-04-02T05:24:05.042580+0000 (oldest deadline 2024-04-02T05:24:30.342538+0000) 2024-04-02T05:24:56.657 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:56 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:56.265+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.043628+0000 front 2024-04-02T05:24:15.043547+0000 (oldest deadline 2024-04-02T05:24:37.943436+0000) 2024-04-02T05:24:56.657 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:56 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:56.265+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6822 osd.2 since back 2024-04-02T05:24:20.244434+0000 front 2024-04-02T05:24:20.244508+0000 (oldest deadline 2024-04-02T05:24:45.544290+0000) 2024-04-02T05:24:56.657 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:56 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:56.265+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6830 osd.3 since back 2024-04-02T05:24:30.845331+0000 front 2024-04-02T05:24:30.845284+0000 (oldest deadline 2024-04-02T05:24:53.745178+0000) 2024-04-02T05:24:56.927 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:56 smithi192 podman[101995]: 2024-04-02 05:24:56.654588022 +0000 UTC m=+5.491434534 container died d3707b413a43f713fa2c7bb475c6b4f04548786309312c544c1775a22c01d594 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6, RELEASE=squid-6cec793, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-02T05:24:57.224 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:56 smithi192 podman[101995]: 2024-04-02 05:24:56.968157805 +0000 UTC m=+5.805004311 container cleanup d3707b413a43f713fa2c7bb475c6b4f04548786309312c544c1775a22c01d594 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, RELEASE=squid-6cec793, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, ceph=True) 2024-04-02T05:24:57.224 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:56 smithi192 bash[101995]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6 2024-04-02T05:24:57.224 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:57 smithi192 podman[102010]: 2024-04-02 05:24:57.144627379 +0000 UTC m=+0.487129787 container remove d3707b413a43f713fa2c7bb475c6b4f04548786309312c544c1775a22c01d594 (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-6cec793, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-04-02T05:24:57.572 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:57.221+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:05.042528+0000 front 2024-04-02T05:24:05.042580+0000 (oldest deadline 2024-04-02T05:24:30.342538+0000) 2024-04-02T05:24:57.573 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:57.221+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.043628+0000 front 2024-04-02T05:24:15.043547+0000 (oldest deadline 2024-04-02T05:24:37.943436+0000) 2024-04-02T05:24:57.573 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:57.221+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6822 osd.2 since back 2024-04-02T05:24:20.244434+0000 front 2024-04-02T05:24:20.244508+0000 (oldest deadline 2024-04-02T05:24:45.544290+0000) 2024-04-02T05:24:57.573 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:57 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:57.221+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6830 osd.3 since back 2024-04-02T05:24:30.845331+0000 front 2024-04-02T05:24:30.845284+0000 (oldest deadline 2024-04-02T05:24:53.745178+0000) 2024-04-02T05:24:57.875 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:57 smithi192 podman[102102]: 2024-04-02 05:24:57.570465625 +0000 UTC m=+0.100412847 container create 8cc3611e61ecc107a7f9133511fd5252fd7c523b6a9382d8cdb56719710508ff (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=squid-6cec793) 2024-04-02T05:24:57.875 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:57 smithi192 podman[102102]: 2024-04-02 05:24:57.487887397 +0000 UTC m=+0.017834583 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:24:57.875 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:57 smithi192 podman[102102]: 2024-04-02 05:24:57.798704634 +0000 UTC m=+0.328651820 container init 8cc3611e61ecc107a7f9133511fd5252fd7c523b6a9382d8cdb56719710508ff (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6-deactivate, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, RELEASE=squid-6cec793, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True) 2024-04-02T05:24:57.875 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:57 smithi192 podman[102102]: 2024-04-02 05:24:57.801981494 +0000 UTC m=+0.331928677 container start 8cc3611e61ecc107a7f9133511fd5252fd7c523b6a9382d8cdb56719710508ff (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6-deactivate, CEPH_POINT_RELEASE=, RELEASE=squid-6cec793, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-02T05:24:57.875 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:57 smithi192 podman[102102]: 2024-04-02 05:24:57.873129583 +0000 UTC m=+0.403076785 container attach 8cc3611e61ecc107a7f9133511fd5252fd7c523b6a9382d8cdb56719710508ff (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-6cec793, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5) 2024-04-02T05:24:58.177 INFO:journalctl@ceph.osd.6.smithi192.stdout:Apr 02 05:24:58 smithi192 podman[102102]: 2024-04-02 05:24:58.086933548 +0000 UTC m=+0.616880735 container died 8cc3611e61ecc107a7f9133511fd5252fd7c523b6a9382d8cdb56719710508ff (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-6-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=squid-6cec793, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-04-02T05:24:58.670 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:58 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:58.271+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:05.042528+0000 front 2024-04-02T05:24:05.042580+0000 (oldest deadline 2024-04-02T05:24:30.342538+0000) 2024-04-02T05:24:58.670 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:58 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:58.271+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.043628+0000 front 2024-04-02T05:24:15.043547+0000 (oldest deadline 2024-04-02T05:24:37.943436+0000) 2024-04-02T05:24:58.670 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:58 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:58.271+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6822 osd.2 since back 2024-04-02T05:24:20.244434+0000 front 2024-04-02T05:24:20.244508+0000 (oldest deadline 2024-04-02T05:24:45.544290+0000) 2024-04-02T05:24:58.670 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:58 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:58.271+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6830 osd.3 since back 2024-04-02T05:24:30.845331+0000 front 2024-04-02T05:24:30.845284+0000 (oldest deadline 2024-04-02T05:24:53.745178+0000) 2024-04-02T05:24:58.691 DEBUG:teuthology.orchestra.run.smithi192:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.6.service' 2024-04-02T05:24:58.731 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T05:24:58.731 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-04-02T05:24:58.731 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-04-02T05:24:58.731 DEBUG:teuthology.orchestra.run.smithi192:> sudo systemctl stop ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.7 2024-04-02T05:24:58.927 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:58 smithi192 systemd[1]: Stopping Ceph osd.7 for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:24:59.427 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:59 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:59.046+0000 7fd451a08700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T05:24:59.427 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:59 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:59.046+0000 7fd451a08700 -1 osd.7 164 *** Got signal Terminated *** 2024-04-02T05:24:59.427 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:59 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:59.046+0000 7fd451a08700 -1 osd.7 164 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T05:24:59.427 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:59 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:59.236+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:05.042528+0000 front 2024-04-02T05:24:05.042580+0000 (oldest deadline 2024-04-02T05:24:30.342538+0000) 2024-04-02T05:24:59.427 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:59 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:59.236+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.043628+0000 front 2024-04-02T05:24:15.043547+0000 (oldest deadline 2024-04-02T05:24:37.943436+0000) 2024-04-02T05:24:59.428 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:59 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:59.236+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6822 osd.2 since back 2024-04-02T05:24:20.244434+0000 front 2024-04-02T05:24:20.244508+0000 (oldest deadline 2024-04-02T05:24:45.544290+0000) 2024-04-02T05:24:59.428 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:24:59 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:24:59.236+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6830 osd.3 since back 2024-04-02T05:24:30.845331+0000 front 2024-04-02T05:24:30.845284+0000 (oldest deadline 2024-04-02T05:24:53.745178+0000) 2024-04-02T05:25:00.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:25:00 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:25:00.197+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:05.042528+0000 front 2024-04-02T05:24:05.042580+0000 (oldest deadline 2024-04-02T05:24:30.342538+0000) 2024-04-02T05:25:00.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:25:00 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:25:00.197+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.043628+0000 front 2024-04-02T05:24:15.043547+0000 (oldest deadline 2024-04-02T05:24:37.943436+0000) 2024-04-02T05:25:00.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:25:00 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:25:00.197+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6822 osd.2 since back 2024-04-02T05:24:20.244434+0000 front 2024-04-02T05:24:20.244508+0000 (oldest deadline 2024-04-02T05:24:45.544290+0000) 2024-04-02T05:25:00.678 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:25:00 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:25:00.197+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6830 osd.3 since back 2024-04-02T05:24:30.845331+0000 front 2024-04-02T05:24:30.845284+0000 (oldest deadline 2024-04-02T05:24:53.745178+0000) 2024-04-02T05:25:01.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:25:01 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:25:01.212+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:05.042528+0000 front 2024-04-02T05:24:05.042580+0000 (oldest deadline 2024-04-02T05:24:30.342538+0000) 2024-04-02T05:25:01.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:25:01 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:25:01.212+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.043628+0000 front 2024-04-02T05:24:15.043547+0000 (oldest deadline 2024-04-02T05:24:37.943436+0000) 2024-04-02T05:25:01.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:25:01 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:25:01.212+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6822 osd.2 since back 2024-04-02T05:24:20.244434+0000 front 2024-04-02T05:24:20.244508+0000 (oldest deadline 2024-04-02T05:24:45.544290+0000) 2024-04-02T05:25:01.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:25:01 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:25:01.212+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6830 osd.3 since back 2024-04-02T05:24:30.845331+0000 front 2024-04-02T05:24:30.845284+0000 (oldest deadline 2024-04-02T05:24:53.745178+0000) 2024-04-02T05:25:01.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:25:01 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:25:01.212+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.192:6804 osd.4 since back 2024-04-02T05:24:39.646009+0000 front 2024-04-02T05:24:39.646029+0000 (oldest deadline 2024-04-02T05:25:00.746025+0000) 2024-04-02T05:25:02.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:25:02 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:25:02.216+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:05.042528+0000 front 2024-04-02T05:24:05.042580+0000 (oldest deadline 2024-04-02T05:24:30.342538+0000) 2024-04-02T05:25:02.678 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:25:02 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:25:02.216+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.043628+0000 front 2024-04-02T05:24:15.043547+0000 (oldest deadline 2024-04-02T05:24:37.943436+0000) 2024-04-02T05:25:02.678 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:25:02 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:25:02.216+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6822 osd.2 since back 2024-04-02T05:24:20.244434+0000 front 2024-04-02T05:24:20.244508+0000 (oldest deadline 2024-04-02T05:24:45.544290+0000) 2024-04-02T05:25:02.678 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:25:02 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:25:02.216+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6830 osd.3 since back 2024-04-02T05:24:30.845331+0000 front 2024-04-02T05:24:30.845284+0000 (oldest deadline 2024-04-02T05:24:53.745178+0000) 2024-04-02T05:25:02.678 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:25:02 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:25:02.216+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.192:6804 osd.4 since back 2024-04-02T05:24:39.646009+0000 front 2024-04-02T05:24:39.646029+0000 (oldest deadline 2024-04-02T05:25:00.746025+0000) 2024-04-02T05:25:03.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:25:03 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:25:03.219+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6806 osd.0 since back 2024-04-02T05:24:05.042528+0000 front 2024-04-02T05:24:05.042580+0000 (oldest deadline 2024-04-02T05:24:30.342538+0000) 2024-04-02T05:25:03.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:25:03 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:25:03.219+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6814 osd.1 since back 2024-04-02T05:24:15.043628+0000 front 2024-04-02T05:24:15.043547+0000 (oldest deadline 2024-04-02T05:24:37.943436+0000) 2024-04-02T05:25:03.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:25:03 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:25:03.219+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6822 osd.2 since back 2024-04-02T05:24:20.244434+0000 front 2024-04-02T05:24:20.244508+0000 (oldest deadline 2024-04-02T05:24:45.544290+0000) 2024-04-02T05:25:03.677 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:25:03 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:25:03.219+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.86:6830 osd.3 since back 2024-04-02T05:24:30.845331+0000 front 2024-04-02T05:24:30.845284+0000 (oldest deadline 2024-04-02T05:24:53.745178+0000) 2024-04-02T05:25:03.678 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:25:03 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7[92325]: 2024-04-02T05:25:03.219+0000 7fd449ff4700 -1 osd.7 164 heartbeat_check: no reply from 172.21.15.192:6804 osd.4 since back 2024-04-02T05:24:39.646009+0000 front 2024-04-02T05:24:39.646029+0000 (oldest deadline 2024-04-02T05:25:00.746025+0000) 2024-04-02T05:25:04.678 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:25:04 smithi192 podman[102206]: 2024-04-02 05:25:04.356479927 +0000 UTC m=+5.409664432 container died 48f6eb4f415f24c4455e4cf4eecbe54f271944b84df8bd13d5bfc211711232cb (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=squid-6cec793, ceph=True) 2024-04-02T05:25:05.177 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:25:04 smithi192 podman[102206]: 2024-04-02 05:25:04.679868026 +0000 UTC m=+5.733052540 container cleanup 48f6eb4f415f24c4455e4cf4eecbe54f271944b84df8bd13d5bfc211711232cb (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-6cec793, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-04-02T05:25:05.177 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:25:04 smithi192 bash[102206]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7 2024-04-02T05:25:05.177 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:25:04 smithi192 podman[102223]: 2024-04-02 05:25:04.864503396 +0000 UTC m=+0.509274858 container remove 48f6eb4f415f24c4455e4cf4eecbe54f271944b84df8bd13d5bfc211711232cb (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-6cec793, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326) 2024-04-02T05:25:05.603 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:25:05 smithi192 podman[102319]: 2024-04-02 05:25:05.190601771 +0000 UTC m=+0.017872317 image pull 78f830190839e38cac46d0f24741058bd5f6b4363a488caf8a9e060d9857e808 quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T05:25:05.603 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:25:05 smithi192 podman[102319]: 2024-04-02 05:25:05.290427606 +0000 UTC m=+0.117698188 container create 265ef631bf72e443436e8a21709cab43ce5688b36966d0261918924086786cdc (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, RELEASE=squid-6cec793, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326) 2024-04-02T05:25:05.603 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:25:05 smithi192 podman[102319]: 2024-04-02 05:25:05.517912196 +0000 UTC m=+0.345182735 container init 265ef631bf72e443436e8a21709cab43ce5688b36966d0261918924086786cdc (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=squid-6cec793, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326) 2024-04-02T05:25:05.603 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:25:05 smithi192 podman[102319]: 2024-04-02 05:25:05.520936079 +0000 UTC m=+0.348210573 container start 265ef631bf72e443436e8a21709cab43ce5688b36966d0261918924086786cdc (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-6cec793, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-04-02T05:25:05.603 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:25:05 smithi192 podman[102319]: 2024-04-02 05:25:05.60112497 +0000 UTC m=+0.428395522 container attach 265ef631bf72e443436e8a21709cab43ce5688b36966d0261918924086786cdc (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7-deactivate, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=squid-6cec793, CEPH_POINT_RELEASE=) 2024-04-02T05:25:05.927 INFO:journalctl@ceph.osd.7.smithi192.stdout:Apr 02 05:25:05 smithi192 podman[102319]: 2024-04-02 05:25:05.811505307 +0000 UTC m=+0.638775838 container died 265ef631bf72e443436e8a21709cab43ce5688b36966d0261918924086786cdc (image=quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-osd-7-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, RELEASE=squid-6cec793, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-02T05:25:06.433 DEBUG:teuthology.orchestra.run.smithi192:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@osd.7.service' 2024-04-02T05:25:06.472 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T05:25:06.472 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-04-02T05:25:06.472 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-04-02T05:25:06.472 DEBUG:teuthology.orchestra.run.smithi192:> sudo systemctl stop ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@prometheus.a 2024-04-02T05:25:06.678 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:25:06 smithi192 systemd[1]: Stopping Ceph prometheus.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:25:07.046 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:25:06 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:25:06.784Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-02T05:25:07.046 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:25:06 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:25:06.785Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-02T05:25:07.046 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:25:06 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:25:06.785Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-02T05:25:07.046 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:25:06 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:25:06.785Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-02T05:25:07.046 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:25:06 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:25:06.785Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-02T05:25:07.046 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:25:06 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:25:06.785Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-02T05:25:07.046 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:25:06 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:25:06.785Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-02T05:25:07.046 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:25:06 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:25:06.785Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-02T05:25:07.046 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:25:06 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:25:06.785Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-02T05:25:07.047 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:25:06 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:25:06.786Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-02T05:25:07.047 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:25:06 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:25:06.786Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-02T05:25:07.047 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:25:06 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a[64406]: ts=2024-04-02T05:25:06.786Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-02T05:25:07.047 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:25:06 smithi192 podman[102425]: 2024-04-02 05:25:06.79523586 +0000 UTC m=+0.106152011 container died 8378fcdb60e3d8fa3b97b388d33d409f20060c23ebb9c6d438a55a8b4bf207cd (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T05:25:07.412 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:25:07 smithi192 podman[102425]: 2024-04-02 05:25:07.14303459 +0000 UTC m=+0.453950744 container cleanup 8378fcdb60e3d8fa3b97b388d33d409f20060c23ebb9c6d438a55a8b4bf207cd (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T05:25:07.412 INFO:journalctl@ceph.prometheus.a.smithi192.stdout:Apr 02 05:25:07 smithi192 bash[102425]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-prometheus-a 2024-04-02T05:25:07.557 DEBUG:teuthology.orchestra.run.smithi192:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@prometheus.a.service' 2024-04-02T05:25:07.593 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T05:25:07.593 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-04-02T05:25:07.594 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 --force --keep-logs 2024-04-02T05:25:09.312 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:25:09 smithi086 systemd[1]: Stopping Ceph node-exporter.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:25:09.582 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:25:09 smithi086 podman[131624]: 2024-04-02 05:25:09.3368166 +0000 UTC m=+0.106154192 container died 05a844569d7b30fcacd71836eb9b14c5b7278d60f158c60f9ed8f4e80c627c0b (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-02T05:25:09.879 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:25:09 smithi086 podman[131624]: 2024-04-02 05:25:09.668094285 +0000 UTC m=+0.437431875 container cleanup 05a844569d7b30fcacd71836eb9b14c5b7278d60f158c60f9ed8f4e80c627c0b (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-02T05:25:09.880 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:25:09 smithi086 bash[131624]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a 2024-04-02T05:25:10.146 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:25:09 smithi086 podman[131638]: 2024-04-02 05:25:09.877373499 +0000 UTC m=+0.544759902 container remove 05a844569d7b30fcacd71836eb9b14c5b7278d60f158c60f9ed8f4e80c627c0b (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-02T05:25:10.146 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:25:09 smithi086 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-02T05:25:10.146 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:25:10 smithi086 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-02T05:25:10.146 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:25:10 smithi086 systemd[1]: Stopped Ceph node-exporter.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:25:10.146 INFO:journalctl@ceph.node-exporter.a.smithi086.stdout:Apr 02 05:25:10 smithi086 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@node-exporter.a.service: Consumed 3.102s CPU time. 2024-04-02T05:25:10.582 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:25:10 smithi086 systemd[1]: Stopping Ceph alertmanager.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:25:10.892 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:25:10 smithi086 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a[79924]: ts=2024-04-02T05:25:10.580Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-02T05:25:10.892 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:25:10 smithi086 podman[131752]: 2024-04-02 05:25:10.631999153 +0000 UTC m=+0.161364377 container died 660ea62bb6b98412a61f27d504bb177e33961057c05751b11bdbf6eab8043b60 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T05:25:11.251 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:25:10 smithi086 podman[131752]: 2024-04-02 05:25:10.989365802 +0000 UTC m=+0.518731025 container cleanup 660ea62bb6b98412a61f27d504bb177e33961057c05751b11bdbf6eab8043b60 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T05:25:11.251 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:25:10 smithi086 bash[131752]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a 2024-04-02T05:25:11.583 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:25:11 smithi086 podman[131766]: 2024-04-02 05:25:11.249070501 +0000 UTC m=+0.609413960 container remove 660ea62bb6b98412a61f27d504bb177e33961057c05751b11bdbf6eab8043b60 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T05:25:11.583 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:25:11 smithi086 podman[131766]: 2024-04-02 05:25:11.333226334 +0000 UTC m=+0.693569793 volume remove 3bada234eed676398a8e78b3b95a18f2c4b3e6f59da5c78155f2cd6758bd17ca 2024-04-02T05:25:11.583 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:25:11 smithi086 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@alertmanager.a.service: Deactivated successfully. 2024-04-02T05:25:11.583 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:25:11 smithi086 systemd[1]: Stopped Ceph alertmanager.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:25:11.583 INFO:journalctl@ceph.alertmanager.a.smithi086.stdout:Apr 02 05:25:11 smithi086 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@alertmanager.a.service: Consumed 1.475s CPU time. 2024-04-02T05:25:34.355 DEBUG:teuthology.orchestra.run.smithi192:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 --force --keep-logs 2024-04-02T05:25:35.988 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:25:35 smithi192 systemd[1]: Stopping Ceph node-exporter.b for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:25:36.397 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:25:36 smithi192 podman[102838]: 2024-04-02 05:25:36.155326464 +0000 UTC m=+0.106287313 container died 1a7d6788d58bdb0973997a9b7fde482a8eee8f4ac65cdf6fbe64b1851ff30451 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-02T05:25:36.677 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:25:36 smithi192 podman[102838]: 2024-04-02 05:25:36.561403485 +0000 UTC m=+0.512364326 container cleanup 1a7d6788d58bdb0973997a9b7fde482a8eee8f4ac65cdf6fbe64b1851ff30451 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-02T05:25:36.677 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:25:36 smithi192 bash[102838]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b 2024-04-02T05:25:37.086 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:25:36 smithi192 podman[102852]: 2024-04-02 05:25:36.787662148 +0000 UTC m=+0.637161988 container remove 1a7d6788d58bdb0973997a9b7fde482a8eee8f4ac65cdf6fbe64b1851ff30451 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-02T05:25:37.086 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:25:36 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-02T05:25:37.086 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:25:36 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-02T05:25:37.086 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:25:36 smithi192 systemd[1]: Stopped Ceph node-exporter.b for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:25:37.086 INFO:journalctl@ceph.node-exporter.b.smithi192.stdout:Apr 02 05:25:36 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@node-exporter.b.service: Consumed 3.070s CPU time. 2024-04-02T05:25:37.336 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:25:37 smithi192 systemd[1]: Stopping Ceph grafana.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4... 2024-04-02T05:25:37.337 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:25:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=server t=2024-04-02T05:25:37.323777603Z level=info msg="Shutdown started" reason="System signal: terminated" 2024-04-02T05:25:37.337 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:25:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=ticker t=2024-04-02T05:25:37.324242474Z level=info msg=stopped last_tick=2024-04-02T05:25:30Z 2024-04-02T05:25:37.337 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:25:37 smithi192 ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a[66369]: logger=tracing t=2024-04-02T05:25:37.324288995Z level=info msg="Closing tracing" 2024-04-02T05:25:37.601 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:25:37 smithi192 podman[102959]: 2024-04-02 05:25:37.334478049 +0000 UTC m=+0.112638690 container died 259af9e63aa540a05375d22d0bca11b8d3b31d70ff20e785000f31710ecad116 (image=quay.io/ceph/grafana:9.4.12, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a, maintainer=Grafana Labs ) 2024-04-02T05:25:37.927 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:25:37 smithi192 podman[102959]: 2024-04-02 05:25:37.682020547 +0000 UTC m=+0.460181229 container cleanup 259af9e63aa540a05375d22d0bca11b8d3b31d70ff20e785000f31710ecad116 (image=quay.io/ceph/grafana:9.4.12, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a, maintainer=Grafana Labs ) 2024-04-02T05:25:37.927 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:25:37 smithi192 bash[102959]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a 2024-04-02T05:25:38.283 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:25:37 smithi192 podman[102972]: 2024-04-02 05:25:37.958155564 +0000 UTC m=+0.621939569 container remove 259af9e63aa540a05375d22d0bca11b8d3b31d70ff20e785000f31710ecad116 (image=quay.io/ceph/grafana:9.4.12, name=ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4-grafana-a, maintainer=Grafana Labs ) 2024-04-02T05:25:38.283 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:25:38 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@grafana.a.service: Deactivated successfully. 2024-04-02T05:25:38.283 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:25:38 smithi192 systemd[1]: Stopped Ceph grafana.a for 8862e8fe-f0ab-11ee-b647-cb9ed24678a4. 2024-04-02T05:25:38.283 INFO:journalctl@ceph.grafana.a.smithi192.stdout:Apr 02 05:25:38 smithi192 systemd[1]: ceph-8862e8fe-f0ab-11ee-b647-cb9ed24678a4@grafana.a.service: Consumed 10.726s CPU time. 2024-04-02T05:25:49.758 DEBUG:teuthology.orchestra.run.smithi086:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-02T05:25:49.794 DEBUG:teuthology.orchestra.run.smithi192:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-02T05:25:49.826 INFO:tasks.cephadm:Archiving crash dumps... 2024-04-02T05:25:49.829 DEBUG:teuthology.misc:Transferring archived files from smithi086:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/crash to /home/teuthworker/archive/teuthology-2024-04-01_21:08:08-orch-squid-distro-default-smithi/7634859/remote/smithi086/crash 2024-04-02T05:25:49.830 DEBUG:teuthology.orchestra.run.smithi086:> sudo tar c -f - -C /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/crash -- . 2024-04-02T05:25:49.883 INFO:teuthology.orchestra.run.smithi086.stderr:tar: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/crash: Cannot open: No such file or directory 2024-04-02T05:25:49.883 INFO:teuthology.orchestra.run.smithi086.stderr:tar: Error is not recoverable: exiting now 2024-04-02T05:25:49.885 DEBUG:teuthology.misc:Transferring archived files from smithi192:/var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/crash to /home/teuthworker/archive/teuthology-2024-04-01_21:08:08-orch-squid-distro-default-smithi/7634859/remote/smithi192/crash 2024-04-02T05:25:49.886 DEBUG:teuthology.orchestra.run.smithi192:> sudo tar c -f - -C /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/crash -- . 2024-04-02T05:25:49.926 INFO:teuthology.orchestra.run.smithi192.stderr:tar: /var/lib/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/crash: Cannot open: No such file or directory 2024-04-02T05:25:49.926 INFO:teuthology.orchestra.run.smithi192.stderr:tar: Error is not recoverable: exiting now 2024-04-02T05:25:49.927 INFO:tasks.cephadm:Checking cluster log for badness... 2024-04-02T05:25:49.928 DEBUG:teuthology.orchestra.run.smithi086:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-02T05:25:49.960 INFO:teuthology.orchestra.run.smithi086.stdout:2024-04-02T04:58:27.647337+0000 mon.a (mon.0) 946 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-02T05:25:49.960 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-04-02T05:25:49.960 DEBUG:teuthology.orchestra.run.smithi086:> sudo egrep '\[SEC\]' /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-02T05:25:50.033 DEBUG:teuthology.orchestra.run.smithi086:> sudo egrep '\[ERR\]' /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-02T05:25:50.104 DEBUG:teuthology.orchestra.run.smithi086:> sudo egrep '\[WRN\]' /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-02T05:25:50.173 INFO:teuthology.orchestra.run.smithi086.stdout:2024-04-02T04:58:27.647337+0000 mon.a (mon.0) 946 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-02T05:25:50.173 INFO:tasks.cephadm:Compressing logs... 2024-04-02T05:25:50.173 DEBUG:teuthology.orchestra.run.smithi086:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-02T05:25:50.216 DEBUG:teuthology.orchestra.run.smithi192:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-02T05:25:50.242 INFO:teuthology.orchestra.run.smithi086.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-02T05:25:50.242 INFO:teuthology.orchestra.run.smithi086.stderr:: No such file or directory 2024-04-02T05:25:50.243 INFO:teuthology.orchestra.run.smithi086.stderr:gzip -5 --verbose -- /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-mon.a.log 2024-04-02T05:25:50.243 INFO:teuthology.orchestra.run.smithi086.stderr:gzip -5 --verbose -- /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph.audit.log 2024-04-02T05:25:50.243 INFO:teuthology.orchestra.run.smithi086.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-osd.1.log 2024-04-02T05:25:50.243 INFO:teuthology.orchestra.run.smithi086.stderr:/var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-mon.a.log: /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-osd.3.log 2024-04-02T05:25:50.243 INFO:teuthology.orchestra.run.smithi086.stderr:/var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-mgr.y.log 2024-04-02T05:25:50.246 INFO:teuthology.orchestra.run.smithi086.stderr:/var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-client.rgw.foo.smithi086.bxzxlp.log 2024-04-02T05:25:50.246 INFO:teuthology.orchestra.run.smithi086.stderr:/var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/tcmu-runner.log 2024-04-02T05:25:50.246 INFO:teuthology.orchestra.run.smithi086.stderr:/var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-client.rgw.foo.smithi086.bxzxlp.log: 76.6% -- replaced with /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-client.rgw.foo.smithi086.bxzxlp.log.gz 2024-04-02T05:25:50.246 INFO:teuthology.orchestra.run.smithi086.stderr:gzip -5 --verbose -- /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-osd.0.log 2024-04-02T05:25:50.247 INFO:teuthology.orchestra.run.smithi086.stderr:/var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/tcmu-runner.log: 90.3% -- replaced with /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/tcmu-runner.log.gz 2024-04-02T05:25:50.247 INFO:teuthology.orchestra.run.smithi086.stderr:gzip -5 --verbose -- /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph.cephadm.log 2024-04-02T05:25:50.250 INFO:teuthology.orchestra.run.smithi192.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-02T05:25:50.251 INFO:teuthology.orchestra.run.smithi192.stderr:: No such file or directory 2024-04-02T05:25:50.251 INFO:teuthology.orchestra.run.smithi192.stderr:gzip -5 --verbose -- /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-osd.6.log 2024-04-02T05:25:50.251 INFO:teuthology.orchestra.run.smithi192.stderr:gzip -5 --verbose -- /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph.audit.log 2024-04-02T05:25:50.251 INFO:teuthology.orchestra.run.smithi192.stderr:/var/log/ceph/cephadm.log: /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-osd.4.log 2024-04-02T05:25:50.251 INFO:teuthology.orchestra.run.smithi192.stderr:/var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-osd.7.log 2024-04-02T05:25:50.251 INFO:teuthology.orchestra.run.smithi192.stderr:gzip -5 --verbose -- /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-osd.5.log 2024-04-02T05:25:50.251 INFO:teuthology.orchestra.run.smithi192.stderr:/var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-osd.4.log: /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-mgr.x.log 2024-04-02T05:25:50.251 INFO:teuthology.orchestra.run.smithi192.stderr:/var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-client.rgw.foo.smithi192.pmlbot.log 2024-04-02T05:25:50.251 INFO:teuthology.orchestra.run.smithi192.stderr:/var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph.cephadm.log 2024-04-02T05:25:50.251 INFO:teuthology.orchestra.run.smithi192.stderr:/var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-client.rgw.foo.smithi192.pmlbot.log: 77.0% -- replaced with /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-client.rgw.foo.smithi192.pmlbot.log.gz 2024-04-02T05:25:50.251 INFO:teuthology.orchestra.run.smithi192.stderr:gzip -5 --verbose -- /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph.log 2024-04-02T05:25:50.251 INFO:teuthology.orchestra.run.smithi192.stderr:/var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph.cephadm.log: 91.9% -- replaced with /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph.cephadm.log.gz 2024-04-02T05:25:50.251 INFO:teuthology.orchestra.run.smithi192.stderr:gzip -5 --verbose -- /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-volume.log 2024-04-02T05:25:50.252 INFO:teuthology.orchestra.run.smithi192.stderr:/var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph.log: 91.0% -- replaced with /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph.audit.log.gz 2024-04-02T05:25:50.253 INFO:teuthology.orchestra.run.smithi086.stderr:/var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph.log 2024-04-02T05:25:50.253 INFO:teuthology.orchestra.run.smithi192.stderr: 89.0% -- replaced with /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph.log.gz 2024-04-02T05:25:50.254 INFO:teuthology.orchestra.run.smithi192.stderr:gzip -5 --verbose -- /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-mon.b.log 2024-04-02T05:25:50.255 INFO:teuthology.orchestra.run.smithi086.stderr:/var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph.cephadm.log: 94.4% -- replaced with /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph.audit.log.gz 2024-04-02T05:25:50.255 INFO:teuthology.orchestra.run.smithi086.stderr:gzip -5 --verbose -- /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-osd.2.log 2024-04-02T05:25:50.255 INFO:teuthology.orchestra.run.smithi086.stderr:/var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph.log: 95.0% -- replaced with /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph.cephadm.log.gz 2024-04-02T05:25:50.258 INFO:teuthology.orchestra.run.smithi086.stderr: 92.4% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-02T05:25:50.258 INFO:teuthology.orchestra.run.smithi086.stderr:gzip -5 --verbose -- /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-volume.log 2024-04-02T05:25:50.264 INFO:teuthology.orchestra.run.smithi086.stderr:/var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-osd.2.log: 93.8% -- replaced with /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph.log.gz 2024-04-02T05:25:50.264 INFO:teuthology.orchestra.run.smithi086.stderr:gzip -5 --verbose -- /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-mon.c.log 2024-04-02T05:25:50.271 INFO:teuthology.orchestra.run.smithi192.stderr:/var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-volume.log: /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-mon.b.log: 91.2% -- replaced with /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-volume.log.gz 2024-04-02T05:25:50.274 INFO:teuthology.orchestra.run.smithi192.stderr: 90.9% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-02T05:25:50.280 INFO:teuthology.orchestra.run.smithi086.stderr:/var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-volume.log: /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-mon.c.log: 91.1% -- replaced with /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-volume.log.gz 2024-04-02T05:25:50.400 INFO:teuthology.orchestra.run.smithi192.stderr: 89.7% -- replaced with /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-mgr.x.log.gz 2024-04-02T05:25:50.747 INFO:teuthology.orchestra.run.smithi086.stderr: 89.1% -- replaced with /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-mgr.y.log.gz 2024-04-02T05:25:50.925 INFO:teuthology.orchestra.run.smithi086.stderr: 92.4% -- replaced with /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-mon.c.log.gz 2024-04-02T05:25:51.184 INFO:teuthology.orchestra.run.smithi192.stderr: 92.3% -- replaced with /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-mon.b.log.gz 2024-04-02T05:25:51.766 INFO:teuthology.orchestra.run.smithi086.stderr: 93.2% -- replaced with /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-osd.2.log.gz 2024-04-02T05:25:51.896 INFO:teuthology.orchestra.run.smithi192.stderr: 93.0% -- replaced with /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-osd.6.log.gz 2024-04-02T05:25:52.148 INFO:teuthology.orchestra.run.smithi192.stderr: 93.6% -- replaced with /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-osd.7.log.gz 2024-04-02T05:25:52.164 INFO:teuthology.orchestra.run.smithi086.stderr: 91.0% -- replaced with /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-mon.a.log.gz 2024-04-02T05:25:52.299 INFO:teuthology.orchestra.run.smithi192.stderr: 93.4% -- replaced with /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-osd.4.log.gz 2024-04-02T05:25:52.407 INFO:teuthology.orchestra.run.smithi086.stderr: 93.5% -- replaced with /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-osd.1.log.gz 2024-04-02T05:25:52.448 INFO:teuthology.orchestra.run.smithi192.stderr: 93.3% -- replaced with /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-osd.5.log.gz 2024-04-02T05:25:52.450 INFO:teuthology.orchestra.run.smithi192.stderr: 2024-04-02T05:25:52.450 INFO:teuthology.orchestra.run.smithi192.stderr:real 0m2.220s 2024-04-02T05:25:52.450 INFO:teuthology.orchestra.run.smithi192.stderr:user 0m8.635s 2024-04-02T05:25:52.450 INFO:teuthology.orchestra.run.smithi192.stderr:sys 0m0.345s 2024-04-02T05:25:52.503 INFO:teuthology.orchestra.run.smithi086.stderr: 93.4% -- replaced with /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-osd.0.log.gz 2024-04-02T05:25:52.935 INFO:teuthology.orchestra.run.smithi086.stderr: 93.3% -- replaced with /var/log/ceph/8862e8fe-f0ab-11ee-b647-cb9ed24678a4/ceph-osd.3.log.gz 2024-04-02T05:25:52.937 INFO:teuthology.orchestra.run.smithi086.stderr: 2024-04-02T05:25:52.937 INFO:teuthology.orchestra.run.smithi086.stderr:real 0m2.709s 2024-04-02T05:25:52.937 INFO:teuthology.orchestra.run.smithi086.stderr:user 0m11.342s 2024-04-02T05:25:52.937 INFO:teuthology.orchestra.run.smithi086.stderr:sys 0m0.408s 2024-04-02T05:25:52.938 INFO:tasks.cephadm:Archiving logs... 2024-04-02T05:25:52.938 DEBUG:teuthology.misc:Transferring archived files from smithi086:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-01_21:08:08-orch-squid-distro-default-smithi/7634859/remote/smithi086/log 2024-04-02T05:25:52.939 DEBUG:teuthology.orchestra.run.smithi086:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-02T05:25:54.460 DEBUG:teuthology.misc:Transferring archived files from smithi192:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-01_21:08:08-orch-squid-distro-default-smithi/7634859/remote/smithi192/log 2024-04-02T05:25:54.461 DEBUG:teuthology.orchestra.run.smithi192:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-02T05:25:55.727 INFO:tasks.cephadm:Removing cluster... 2024-04-02T05:25:55.728 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 --force 2024-04-02T05:25:56.021 DEBUG:teuthology.orchestra.run.smithi192:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 8862e8fe-f0ab-11ee-b647-cb9ed24678a4 --force 2024-04-02T05:25:56.313 INFO:tasks.cephadm:Removing cephadm ... 2024-04-02T05:25:56.313 DEBUG:teuthology.orchestra.run.smithi086:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-02T05:25:56.328 DEBUG:teuthology.orchestra.run.smithi192:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-02T05:25:56.343 INFO:tasks.cephadm:Teardown complete 2024-04-02T05:25:56.343 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-04-02T05:25:56.356 INFO:teuthology.task.clock:Checking final clock skew... 2024-04-02T05:25:56.356 DEBUG:teuthology.orchestra.run.smithi086:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-02T05:25:56.370 DEBUG:teuthology.orchestra.run.smithi192:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-02T05:25:56.382 INFO:teuthology.orchestra.run.smithi086.stderr:bash: line 1: ntpq: command not found 2024-04-02T05:25:56.387 INFO:teuthology.orchestra.run.smithi086.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-02T05:25:56.387 INFO:teuthology.orchestra.run.smithi086.stdout:=============================================================================== 2024-04-02T05:25:56.387 INFO:teuthology.orchestra.run.smithi086.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 28 -844us[ -844us] +/- 72ms 2024-04-02T05:25:56.387 INFO:teuthology.orchestra.run.smithi086.stdout:^* hv02.front.sepia.ceph.com 3 6 377 29 -2519us[-2570us] +/- 60ms 2024-04-02T05:25:56.387 INFO:teuthology.orchestra.run.smithi086.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 27 +4248us[+4248us] +/- 79ms 2024-04-02T05:25:56.387 INFO:teuthology.orchestra.run.smithi086.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T05:25:56.397 INFO:teuthology.orchestra.run.smithi192.stderr:bash: line 1: ntpq: command not found 2024-04-02T05:25:56.400 INFO:teuthology.orchestra.run.smithi192.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-02T05:25:56.401 INFO:teuthology.orchestra.run.smithi192.stdout:=============================================================================== 2024-04-02T05:25:56.401 INFO:teuthology.orchestra.run.smithi192.stdout:^+ hv01.front.sepia.ceph.com 2 7 377 90 -809us[ -895us] +/- 71ms 2024-04-02T05:25:56.401 INFO:teuthology.orchestra.run.smithi192.stdout:^* hv02.front.sepia.ceph.com 3 7 377 31 -2444us[-2526us] +/- 60ms 2024-04-02T05:25:56.401 INFO:teuthology.orchestra.run.smithi192.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 28 +4249us[+4249us] +/- 79ms 2024-04-02T05:25:56.401 INFO:teuthology.orchestra.run.smithi192.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T05:25:56.402 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-02T05:25:56.411 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-02T05:25:56.412 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-02T05:25:56.425 DEBUG:teuthology.orchestra.run.smithi086:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-02T05:25:56.519 DEBUG:teuthology.orchestra.run.smithi192:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-02T05:25:56.600 DEBUG:teuthology.orchestra.run.smithi086:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-02T05:25:56.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T05:25:56.631 DEBUG:teuthology.orchestra.run.smithi192:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-02T05:25:56.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T05:25:56.662 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-02T05:25:56.674 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-02T05:25:56.688 INFO:teuthology.task.internal:Duration was 2808.210328 seconds 2024-04-02T05:25:56.689 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-02T05:25:56.703 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-02T05:25:56.703 DEBUG:teuthology.orchestra.run.smithi086:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-02T05:25:56.706 DEBUG:teuthology.orchestra.run.smithi192:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-02T05:25:56.747 INFO:teuthology.orchestra.run.smithi086.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-02T05:25:56.748 INFO:teuthology.orchestra.run.smithi192.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-02T05:25:57.127 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-02T05:25:57.127 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi086.front.sepia.ceph.com 2024-04-02T05:25:57.127 DEBUG:teuthology.orchestra.run.smithi086:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-02T05:25:57.187 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi192.front.sepia.ceph.com 2024-04-02T05:25:57.187 DEBUG:teuthology.orchestra.run.smithi192:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-02T05:25:57.216 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-02T05:25:57.216 DEBUG:teuthology.orchestra.run.smithi086:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-02T05:25:57.231 DEBUG:teuthology.orchestra.run.smithi192:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-02T05:25:57.286 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-02T05:25:57.286 DEBUG:teuthology.orchestra.run.smithi086:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-02T05:25:57.300 DEBUG:teuthology.orchestra.run.smithi192:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-02T05:25:57.894 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-02T05:25:57.905 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-02T05:25:57.905 DEBUG:teuthology.orchestra.run.smithi086:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-02T05:25:57.932 DEBUG:teuthology.orchestra.run.smithi192:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-02T05:25:57.960 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-02T05:25:57.973 DEBUG:teuthology.orchestra.run.smithi086:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-02T05:25:57.975 DEBUG:teuthology.orchestra.run.smithi192:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-02T05:25:57.998 INFO:teuthology.orchestra.run.smithi086.stdout:kernel.core_pattern = core 2024-04-02T05:25:58.025 INFO:teuthology.orchestra.run.smithi192.stdout:kernel.core_pattern = core 2024-04-02T05:25:58.048 DEBUG:teuthology.orchestra.run.smithi086:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-02T05:25:58.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T05:25:58.077 DEBUG:teuthology.orchestra.run.smithi192:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-02T05:25:58.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T05:25:58.103 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-02T05:25:58.118 INFO:teuthology.task.internal:Transferring archived files... 2024-04-02T05:25:58.119 DEBUG:teuthology.misc:Transferring archived files from smithi086:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-01_21:08:08-orch-squid-distro-default-smithi/7634859/remote/smithi086 2024-04-02T05:25:58.119 DEBUG:teuthology.orchestra.run.smithi086:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-02T05:25:58.173 DEBUG:teuthology.misc:Transferring archived files from smithi192:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-01_21:08:08-orch-squid-distro-default-smithi/7634859/remote/smithi192 2024-04-02T05:25:58.173 DEBUG:teuthology.orchestra.run.smithi192:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-02T05:25:58.220 INFO:teuthology.task.internal:Removing archive directory... 2024-04-02T05:25:58.220 DEBUG:teuthology.orchestra.run.smithi086:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-02T05:25:58.223 DEBUG:teuthology.orchestra.run.smithi192:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-02T05:25:58.266 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-02T05:25:58.281 INFO:teuthology.task.internal:Not uploading archives. 2024-04-02T05:25:58.281 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-02T05:25:58.297 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-02T05:25:58.297 DEBUG:teuthology.orchestra.run.smithi086:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-02T05:25:58.299 DEBUG:teuthology.orchestra.run.smithi192:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-02T05:25:58.311 INFO:teuthology.orchestra.run.smithi086.stdout: 262166 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 2 05:25 /home/ubuntu/cephtest 2024-04-02T05:25:58.321 INFO:teuthology.orchestra.run.smithi192.stdout: 262166 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 2 05:25 /home/ubuntu/cephtest 2024-04-02T05:25:58.322 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-02T05:25:58.333 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-02T05:25:58.356 INFO:teuthology.nuke:Checking targets against current locks 2024-04-02T05:25:58.385 DEBUG:teuthology.nuke:shortname: smithi086 2024-04-02T05:25:58.386 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-02T05:25:58.411 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi086.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-01_21:08:08-orch-squid-distro-default-smithi/7634859', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-02 04:30:39.916834', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBD6U1xsGHi5TYjDNNLsRfTlkPPq3bmTd/DfNTpWlI/gq/At+1k54Bp4jzuZ+GEic4wDmCv31U+JoPc4HUo7Llgo='} 2024-04-02T05:25:58.412 DEBUG:teuthology.nuke:shortname: smithi192 2024-04-02T05:25:58.413 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-02T05:25:58.440 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi192.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-01_21:08:08-orch-squid-distro-default-smithi/7634859', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-02 04:30:39.915163', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHRRJNkbXvbeEt5uCG7WHBMoLFh1on7lkBXYMBfbltGylPGKF3sF4X+1aN2bwtLdAcXFQ1RRPJ2ckWMakwO0nwY='} 2024-04-02T05:25:58.471 INFO:teuthology.orchestra.console.smithi086:Power off 2024-04-02T05:25:58.472 DEBUG:teuthology.orchestra.console.smithi086:pexpect command: ipmitool -H smithi086.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-02T05:25:58.497 DEBUG:teuthology.orchestra.console.smithi086:power off output: Chassis Power Control: Down/Off 2024-04-02T05:25:58.498 DEBUG:teuthology.orchestra.console.smithi086:pexpect command: ipmitool -H smithi086.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-02T05:25:58.513 INFO:teuthology.orchestra.console.smithi192:Power off 2024-04-02T05:25:58.513 DEBUG:teuthology.orchestra.console.smithi192:pexpect command: ipmitool -H smithi192.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-02T05:25:58.523 DEBUG:teuthology.orchestra.console.smithi086:check power output: Chassis Power is on 2024-04-02T05:25:58.535 DEBUG:teuthology.orchestra.console.smithi192:power off output: Chassis Power Control: Down/Off 2024-04-02T05:25:58.535 DEBUG:teuthology.orchestra.console.smithi192:pexpect command: ipmitool -H smithi192.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-02T05:25:58.561 DEBUG:teuthology.orchestra.console.smithi192:check power output: Chassis Power is on 2024-04-02T05:26:02.525 DEBUG:teuthology.orchestra.console.smithi086:pexpect command: ipmitool -H smithi086.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-02T05:26:02.562 DEBUG:teuthology.orchestra.console.smithi192:pexpect command: ipmitool -H smithi192.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-02T05:26:02.635 DEBUG:teuthology.orchestra.console.smithi086:check power output: Chassis Power is on 2024-04-02T05:26:02.671 DEBUG:teuthology.orchestra.console.smithi192:check power output: Chassis Power is on 2024-04-02T05:26:06.640 DEBUG:teuthology.orchestra.console.smithi086:pexpect command: ipmitool -H smithi086.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-02T05:26:06.672 DEBUG:teuthology.orchestra.console.smithi192:pexpect command: ipmitool -H smithi192.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-02T05:26:06.749 DEBUG:teuthology.orchestra.console.smithi086:check power output: Chassis Power is on 2024-04-02T05:26:06.782 DEBUG:teuthology.orchestra.console.smithi192:check power output: Chassis Power is off 2024-04-02T05:26:06.882 INFO:teuthology.orchestra.console.smithi192:Power off completed 2024-04-02T05:26:10.750 DEBUG:teuthology.orchestra.console.smithi086:pexpect command: ipmitool -H smithi086.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-02T05:26:10.861 DEBUG:teuthology.orchestra.console.smithi086:check power output: Chassis Power is off 2024-04-02T05:26:10.962 INFO:teuthology.orchestra.console.smithi086:Power off completed 2024-04-02T05:26:11.066 INFO:teuthology.run:Summary data: description: orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/repo_digest 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} duration: 2808.2103276252747 failure_reason: '"2024-04-02T04:58:27.647337+0000 mon.a (mon.0) 946 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON)" in cluster log' owner: scheduled_teuthology@teuthology success: false 2024-04-02T05:26:11.066 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-02T05:26:11.152 INFO:teuthology.run:FAIL